mirror of
https://github.com/TheFunny/TelegramTwitterMediaBot.git
synced 2026-09-23 23:32:05 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a92b12f633
|
||
|
|
74e3b7593c
|
||
|
|
2faccaac42
|
||
|
|
f4e60d8946
|
||
|
|
3006dcd98c
|
||
|
|
8e8acdd859
|
||
|
|
8b9dd963e1
|
||
|
|
e83d48f1f7
|
||
|
|
7c55b26731
|
||
|
|
950db48a13
|
||
|
|
32ea8ec6ca
|
||
|
|
62dc033452
|
||
|
|
2801aaa39c
|
||
|
|
93d47752cf
|
||
|
|
6fd4edb3c5
|
||
|
|
7c5afce0b4
|
||
|
|
caf41183a4
|
||
|
|
b6ae88e869
|
||
|
|
3dff9a2473
|
@@ -28,5 +28,9 @@ LICENSE
|
||||
README.md
|
||||
data/
|
||||
cert/
|
||||
nginx-certs/
|
||||
nginx-vhost.d/
|
||||
nginx-html/
|
||||
nginx-acme/
|
||||
**/target/
|
||||
.idea/
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# Shell scripts must stay LF: CRLF breaks the shebang inside containers.
|
||||
*.sh text eol=lf
|
||||
@@ -12,12 +12,35 @@ env:
|
||||
DOCKERHUB_REPO: yoursfunny/telegram-twitter-media-bot
|
||||
|
||||
jobs:
|
||||
# A tag push and a branch push to the same commit fire two workflow runs;
|
||||
# build only once. Tag runs always build; master runs build only when the
|
||||
# pushed commit is not already tagged (the tag run covers it).
|
||||
should-build:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
build: ${{ steps.check.outputs.build }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: check
|
||||
shell: bash
|
||||
run: |
|
||||
if [ "$GITHUB_REF_TYPE" = "branch" ] && git tag --points-at "$GITHUB_SHA" | grep -q .; then
|
||||
echo "commit already tagged; the tag run builds the image"
|
||||
echo "build=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "build=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
docker:
|
||||
needs: should-build
|
||||
if: needs.should-build.outputs.build == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
uses: docker/metadata-action@v6
|
||||
with:
|
||||
images: ${{ env.DOCKERHUB_REPO }}
|
||||
tags: |
|
||||
@@ -28,22 +51,30 @@ jobs:
|
||||
type=sha
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
uses: docker/setup-qemu-action@v4
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
# Buildkit cache via the GitHub Actions cache backend (uses the
|
||||
# automatic GITHUB_TOKEN, no extra secrets). mode=max keeps every
|
||||
# stage's layers so the cargo-deps and ffmpeg layers are restored
|
||||
# instead of re-downloaded/recompiled. The scope must be pinned to a
|
||||
# fixed string: the gha backend defaults to the current git ref, which
|
||||
# would give every new tag a cold cache on release builds.
|
||||
-
|
||||
name: Build and push
|
||||
uses: docker/build-push-action@v5
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
push: true
|
||||
build-args: |
|
||||
APP_NAME=${{ env.APP_NAME }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha,scope=tgxmb-build
|
||||
cache-to: type=gha,mode=max,scope=tgxmb-build
|
||||
|
||||
@@ -2,6 +2,10 @@
|
||||
__pycache__/
|
||||
cert/
|
||||
data/
|
||||
nginx-certs/
|
||||
nginx-vhost.d/
|
||||
nginx-html/
|
||||
nginx-acme/
|
||||
docker-compose.yml
|
||||
|
||||
.env
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
# Repository Guidelines
|
||||
|
||||
## Project Overview
|
||||
|
||||
Telegram bot (teloxide) that turns post links from X/Twitter, Pixiv, and Bluesky into media messages (images, video, GIF) with the post's title, author, and tags. It supports batch media splitting, retry with persistence, inline queries, forward-channel rebinding with caption templates, and Pixiv ugoira→MP4 transcoding. README and user-facing strings are in Chinese. The project is a Rust port of a Python predecessor (see `queue.rs` comments referencing `utils/task_queue.py`).
|
||||
|
||||
Two-crate Cargo workspace (both v1.0.3, edition 2024, resolver 3):
|
||||
|
||||
- **`crates/x-media`** — library that fetches and normalizes media from the three sites. Pure, no Telegram knowledge.
|
||||
- **`crates/xmedia-bot`** — the bot binary: teloxide dispatcher, SQLite-backed chat state, persistent task queue.
|
||||
|
||||
## Architecture & Data Flow
|
||||
|
||||
```
|
||||
Telegram update → Dispatcher (polling or axum webhook) → dptree branches
|
||||
├─ message → commands (any chat) / URL links (private chat only)
|
||||
├─ inline_query → InlineQueryResult Photo/Video/Mpeg4Gif
|
||||
└─ callback_query → "forward" (copy to channel) / "template|<name>" (apply caption template)
|
||||
```
|
||||
|
||||
Message flow: `message_handler` extracts URLs (from `url`/`text_link` entities, text + caption, deduped) → `x_media::site::fetch(url)` → `Fetched` → builds a `Task` → `send::send_media_sequence` (media groups ≤ 9, caption on first item) or `send::send_animation`. On Telegram URL-fetch failure or size error (`send_batch_via_upload`): download via `x_media::site::download_media` to a temp file (≤ 10 MiB), sniff magic bytes (`sniff_ext`), upload via multipart; oversized items fall back to `fallback_url`. On failure: `enqueue_retry` persists resume-state `Task` into the SQLite queue → single worker leases (120 s lock TTL) → retry with exponential backoff (≤ 30 s, `MAX_RETRIES = 2`) → dead-letter → `notify_failure`. Success → `post_send_actions`: edit-before-forward prompt with inline buttons, or `copy_messages` to the bound forward channel.
|
||||
|
||||
The `x-media` library: `site::fetch(url)` dispatches (in order) twitter → bsky → pixiv via per-site regex `PATTERN` and returns `Ok(None)` for unmatched URLs. `Fetched { source_url, caption, title, media: Vec<Media>, sensitive, … }`; `caption_with(format)` substitutes `{url} {author} {author_url} {title} {tags}`.
|
||||
|
||||
## Key Directories
|
||||
|
||||
| Path | Purpose |
|
||||
|---|---|
|
||||
| `crates/x-media/src/` | Fetch library. `site/mod.rs` = dispatcher + `Fetched`/`FetchError`/`download_media`/`media_size`; `media.rs` = `Media` enum; `examples/fetch.rs` = end-to-end usage sample |
|
||||
| `crates/x-media/src/site/<twitter\|pixiv\|bsky>/` | One directory per site: `mod.rs` (re-exports), `interface.rs` (PATTERN, `enabled()`, `fetch_from_url()`, site struct, `From<SiteStruct> for Fetched`), `model.rs` (serde DTOs). Pixiv adds `api.rs` (auth + transport); twitter adds `auth.rs` (logged-in GraphQL `TweetDetail` fallback for NSFW tweets, gated on `TWITTER_AUTH_TOKEN`) |
|
||||
| `crates/xmedia-bot/src/main.rs` | Entry point: env/log init, queue worker start, pixiv validation, 300 s edit-expiry sweep, dptree handler tree, webhook vs polling dispatch |
|
||||
| `crates/xmedia-bot/src/config.rs` | Manual env parsing into `Config` |
|
||||
| `crates/xmedia-bot/src/handlers.rs` | `Command` enum (teloxide `BotCommands`), message/inline/callback handlers, URL extraction, global statics |
|
||||
| `crates/xmedia-bot/src/state.rs` | `ChatStore`: parking_lot `Mutex<HashMap>` cache + SQLite write-through (`chat_state` table) |
|
||||
| `crates/xmedia-bot/src/queue.rs` | `PersistentTaskQueue`: SQLite-backed single-worker queue (`tasks` table) |
|
||||
| `crates/xmedia-bot/src/send.rs` | Media senders, upload fallback, error classification, queue task handlers |
|
||||
|
||||
## Development Commands
|
||||
|
||||
```bash
|
||||
export TELOXIDE_TOKEN=<token> # required; PIXIV_REFRESH_TOKEN optional (Pixiv disabled without it)
|
||||
cargo run -p xmedia-bot # run the bot (polling by default)
|
||||
cargo run -p x-media --example fetch -- <url> # test a link through the fetch library
|
||||
cargo test --workspace # full test suite (no CI test step exists — run locally)
|
||||
cargo build --release -p xmedia-bot # release build (Dockerfile does this)
|
||||
cargo clippy --workspace --all-targets # lint (Clippy is the configured IDE linter)
|
||||
cargo fmt --check # formatting
|
||||
```
|
||||
|
||||
Docker: `docker build -t tgxmb .` then `docker run --rm -d --name tgxmb --env-file .env -v ./data:/app/data tgxmb`. Runtime requires **ffmpeg** (built into the image).
|
||||
|
||||
## Code Conventions & Common Patterns
|
||||
|
||||
- **No anyhow/thiserror.** Errors are hand-rolled enums with manual `Display`/`source()`/`From` impls: `QueueError` (`Retryable { delay_seconds, payload }` / `Permanent`), `SendError` (Retryable/Permanent), `FetchError` (`Http`/`Json`/`Pixiv`/`NotFound`/`Blocked`), `PixivError`, `Classification`. New errors should follow this pattern.
|
||||
- **Global state via `std::sync::LazyLock` statics**, not DI: `CONFIG`, `CHAT_STORE`, `TASK_QUEUE` in `handlers.rs`; shared reqwest `CLIENT` in `x-media/src/site/mod.rs`. `Bot` is passed/cloned into handlers; queue workers rebuild `Bot::from_env()`.
|
||||
- **Async**: tokio multi-thread runtime (`#[tokio::main]` default). All rusqlite I/O inside `tokio::task::spawn_blocking`. Long loops use `tokio::select!` with `tokio::sync::{watch, Notify}` stop/wake channels. No streams.
|
||||
- **Blocking sync primitives**: `parking_lot::Mutex` for hot caches, `tokio::sync::Mutex` for async-shared state (pixiv token cache), `AtomicBool` for feature gates.
|
||||
- **Site adapter convention** (no trait, no enum dispatch — follow the existing convention): each site module exports `PATTERN: LazyLock<Regex>`, `enabled() -> bool`, `fetch_from_url(url) -> Result<Fetched, FetchError>`; `site/mod.rs` re-exports the site struct and `fetch_once` adds one guarded if-branch. Adding a site = new `site/<name>/{mod.rs,interface.rs,model.rs}` + one branch in `fetch_once`.
|
||||
- **Serde**: per-site `model.rs` are pure `Deserialize` DTOs mirroring API JSON; site structs in `interface.rs` have private fields, a `caption()` builder, and `impl From<SiteStruct> for Fetched`. Persisted payloads use internally-tagged enums (`#[serde(tag = "kind")]` / `type`).
|
||||
- **Naming**: module-per-concern, snake_case files, `CamelCase` types, `snake_case` fns. `//!` module docs and `///` docs on non-obvious logic (syndication token, ugoira encoding, `display_text_range`).
|
||||
- **Retries**: only `x-media::site::fetch` retries (3 attempts, `1 << attempt` backoff, HTTP errors only). Queue retries are explicit `QueueError::Retryable` with computed delay (`retry_delay_seconds`).
|
||||
- Logging via `log` macros (`pretty_env_logger`, level from `RUST_LOG`).
|
||||
|
||||
## Important Files
|
||||
|
||||
| File | Why it matters |
|
||||
|---|---|
|
||||
| `crates/xmedia-bot/src/main.rs` | Startup sequence, webhook vs polling, graceful shutdown (SIGINT via teloxide ctrlc / SIGTERM via `stop_token` for docker, → sweep stop → admin msg → queue stop) |
|
||||
| `crates/xmedia-bot/src/handlers.rs` | `CHAT_STORE`/`TASK_QUEUE`/`CONFIG` singletons (open `data/task_queue.db` **relative to CWD**); command dispatch; URL extraction; retry enqueue |
|
||||
| `crates/xmedia-bot/src/send.rs` | Constants `MAX_MEDIA_GROUP = 9`, `MAX_UPLOAD_BYTES = 10 MiB`; fallback chain; `classify_request_error` |
|
||||
| `crates/x-media/src/site/mod.rs` | Dispatcher, `Fetched`/`FetchError`, shared `CLIENT`, `download_media` (adds `Referer: https://www.pixiv.net/` for `pximg.net` hotlink protection) |
|
||||
| `crates/x-media/src/site/pixiv/api.rs` | OAuth token exchange (hardcoded app client id/secret), access-token cache, ugoira zip→MP4 via ffmpeg in `spawn_blocking` |
|
||||
| `Dockerfile` | Multi-stage: cached dep layer via stub sources + `touch *.rs` mtime hack, static ffmpeg from ffmpeg.martin-riedl.de (`FFMPEG_URL` arg, `unzip -t` integrity check), `debian:bookworm-slim` runtime, entrypoint |
|
||||
| `docker-entrypoint.sh` | Privilege drop: `useradd` with `LOCAL_USER_ID` (default 9001) + `setpriv` (no gosu on bookworm-slim) |
|
||||
| `docker-compose.yml.example` | Deployment env reference (real `docker-compose.yml` is gitignored). Ships nginx-proxy + acme-companion: webhook mode needs TLS termination in front (teloxide's axum listener is HTTP-only; `WEBHOOK_CERT` only feeds `set_webhook`), bot exposes `VIRTUAL_HOST`/`VIRTUAL_PORT` on the shared `proxy` network, no host port; container names `nginx-proxy`/`acme-companion`/`tgxmb`, start order via `depends_on` (proxy → acme → bot) |
|
||||
| `.github/workflows/docker.yml` | CI: build+push to Docker Hub on tag `v*`/master; **no test step**; buildx gha cache (`cache-from`/`cache-to`, scope `tgxmb-build`, `mode=max`) so cargo deps + ffmpeg layers are restored across runs |
|
||||
| `README.md` | Feature docs + command table (Chinese) |
|
||||
|
||||
## Runtime/Tooling Preferences
|
||||
|
||||
- **Rust, stable, edition 2024**, workspace resolver 3. No `rust-version`/MSRV pin, no `rust-toolchain.toml` — recent stable is assumed. No nightly features.
|
||||
- Package manager: **Cargo** (workspace with path dep `x-media` ← `xmedia-bot`). No `[workspace.package]`/shared deps — each crate lists deps independently.
|
||||
- **Two reqwest versions coexist in the lock** (0.12.28 via teloxide, 0.13.3 in x-media) — don't unify casually.
|
||||
- Config is **environment-variable driven** (dotenv loads `.env`, gitignored; no `.env.example` exists). Key vars: `TELOXIDE_TOKEN` (required), `PIXIV_REFRESH_TOKEN`, `TWITTER_AUTH_TOKEN` (optional; x.com `auth_token` cookie — enables the logged-in GraphQL fallback that fetches NSFW tweets syndication withholds), `BOT_ADMIN` (comma-separated ids), `EDIT_MESSAGE_TTL_SECONDS` (default 86400), `WEBHOOK`/`WEBHOOK_URL`/`WEBHOOK_LISTEN`/`WEBHOOK_PORT`/`WEBHOOK_CERT`/`WEBHOOK_SECRET_TOKEN` (webhook mode requires URL/listen/port, `.expect`ed; `WEBHOOK_CERT` is Telegram-facing self-signed validation only — TLS must be terminated by a reverse proxy), `RUST_LOG`, `TELOXIDE_PROXY`, `LOCAL_USER_ID` (entrypoint only).
|
||||
- SQLite via `rusqlite` with `bundled` feature (no system libsqlite needed). DB file `data/task_queue.db` is CWD-relative — run from the workspace root, or `/app` in Docker. Mount `./data` and `./cert` volumes.
|
||||
- `.gitattributes` enforces LF for `*.sh` (CRLF breaks shebangs in containers). `.gitignore`: `.env`, `data/`, `cert/`, `docker-compose.yml`, `/target`, `.idea/`.
|
||||
- Docs are in Chinese; user-facing bot strings too. Keep that convention when editing captions/templates/docs.
|
||||
|
||||
## Testing & QA
|
||||
|
||||
- **~51 tests, all inline `#[cfg(test)] mod tests`** — no `tests/` integration directories. Framework: built-in Rust test + `#[tokio::test]` (dev-deps only in `x-media`: tokio macros/rt-multi-thread, dotenv).
|
||||
- No mocking framework anywhere (no mockito/wiremock/mockall). Conventions: pure-function units (regex parsing, serde round-trips, chunking, retry math) tested synchronously; async tests use real dependencies — file-backed SQLite via `tempfile` (`queue.rs::new_queue()` helper), live network fetches.
|
||||
- Live-network tests exist in `site/twitter/interface.rs` (3), `site/bsky/interface.rs` (2), `site/pixiv/interface.rs`/`api.rs` (env-gated on `PIXIV_REFRESH_TOKEN`/dotenv, skip by early return). Run the full suite with `cargo test --workspace`.
|
||||
- Fixtures are inline `serde_json::json!` builder fns (`fixture()`, `thread_json()`, `illust_json()`), not files. The shared `CLIENT` sets `pool_max_idle_per_host(0)` under `#[cfg(test)]` to avoid cross-runtime `DispatchGone`.
|
||||
- **CI runs no tests** — `.github/workflows/docker.yml` only builds/pushes the image; verification is a local responsibility.
|
||||
- Untested and hard to test without a mock seam: `handlers.rs` (depends directly on teloxide `Bot`); `main.rs`, `config.rs`, `state.rs`; `media.rs`, `lib.rs`, all `model.rs`.
|
||||
- No coverage tracking, no lint gate in CI.
|
||||
Generated
+3
-2
@@ -3296,12 +3296,13 @@ checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"
|
||||
|
||||
[[package]]
|
||||
name = "x-media"
|
||||
version = "1.0.0"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"dotenv",
|
||||
"html-escape",
|
||||
"log",
|
||||
"rand 0.8.6",
|
||||
"regex",
|
||||
"reqwest 0.13.3",
|
||||
"serde",
|
||||
@@ -3314,7 +3315,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "xmedia-bot"
|
||||
version = "1.0.0"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"dotenv",
|
||||
"html-escape",
|
||||
|
||||
+23
-11
@@ -1,12 +1,16 @@
|
||||
# ---------- build stage ----------
|
||||
# rust:1-bookworm (full, not slim) ships the C toolchain needed by
|
||||
# rusqlite's bundled SQLite, plus wget/xz for the ffmpeg download.
|
||||
# rusqlite's bundled SQLite, plus wget/unzip for the ffmpeg download.
|
||||
FROM rust:1-bookworm AS builder
|
||||
|
||||
ARG APP_NAME=telegram-twitter-media-bot
|
||||
# Statically compiled ffmpeg (ugoira MP4 encoding). amd64 by default; override
|
||||
# for other platforms or pin a different johnvansickle build.
|
||||
ARG FFMPEG_URL=https://johnvansickle.com/ffmpeg/releases/ffmpeg-7.0.2-amd64-static.tar.xz
|
||||
# Prebuilt static ffmpeg (glibc-linked, includes libx264) for ugoira MP4
|
||||
# encoding. Served from https://ffmpeg.martin-riedl.de (Cloudflare CDN,
|
||||
# built on Debian 12 — glibc-compatible with the bookworm-slim runtime).
|
||||
# johnvansickle.com throttles datacenter IPs and served garbage from GitHub
|
||||
# runners. `/redirect/latest/` floats to the newest release build; each build
|
||||
# also ships a .sha256. Swap `amd64` for `arm64` when building arm64 images.
|
||||
ARG FFMPEG_URL=https://ffmpeg.martin-riedl.de/redirect/latest/linux/amd64/release/ffmpeg.zip
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
@@ -22,16 +26,24 @@ RUN mkdir -p crates/x-media/src crates/xmedia-bot/src \
|
||||
&& cargo build --release -p xmedia-bot
|
||||
|
||||
# 2. Static ffmpeg next (cached unless FFMPEG_URL changes), so source edits
|
||||
# never re-download it. The johnvansickle tarball has a
|
||||
# `{build}/ffmpeg` layout, so strip one path component.
|
||||
RUN wget -q -O /tmp/ffmpeg.tar.xz "$FFMPEG_URL" \
|
||||
&& tar -xJf /tmp/ffmpeg.tar.xz -C /usr/local/bin --strip-components=1 --wildcards '*/ffmpeg' \
|
||||
&& rm /tmp/ffmpeg.tar.xz \
|
||||
# never re-download it. The zip contains a single `ffmpeg` binary at the
|
||||
# root. `unzip -t` verifies the archive before extraction so a bad
|
||||
# download fails loudly here instead of a cryptic later error.
|
||||
RUN wget -q -O /tmp/ffmpeg.zip "$FFMPEG_URL" \
|
||||
&& unzip -tq /tmp/ffmpeg.zip \
|
||||
&& unzip -q /tmp/ffmpeg.zip -d /usr/local/bin \
|
||||
&& chmod +x /usr/local/bin/ffmpeg \
|
||||
&& rm /tmp/ffmpeg.zip \
|
||||
&& /usr/local/bin/ffmpeg -version >/dev/null
|
||||
|
||||
# 3. Real sources last: only our crates recompile on source changes.
|
||||
# 3. Real sources last: only our crates recompile on source changes. The
|
||||
# COPY preserves host mtimes, which predate the stub artifacts from step 1;
|
||||
# cargo's mtime-based freshness check would otherwise treat the stub build
|
||||
# as up-to-date and never compile the real sources. `touch` forces cargo to
|
||||
# see the real files as newer.
|
||||
COPY crates/ ./crates/
|
||||
RUN cargo build --release -p xmedia-bot
|
||||
RUN find crates -type f -name '*.rs' -exec touch {} + \
|
||||
&& cargo build --release -p xmedia-bot
|
||||
|
||||
# ---------- runtime stage ----------
|
||||
FROM debian:bookworm-slim
|
||||
|
||||
@@ -28,7 +28,52 @@ docker build -t tgxmb .
|
||||
docker run --rm -d --name tgxmb --env-file .env -v ./data:/app/data tgxmb
|
||||
```
|
||||
|
||||
环境变量:`TELOXIDE_TOKEN`(必填)、`PIXIV_REFRESH_TOKEN`、`BOT_ADMIN`、`EDIT_MESSAGE_TTL_SECONDS`、`RUST_LOG`、`WEBHOOK*`。
|
||||
环境变量:`TELOXIDE_TOKEN`(必填)、`PIXIV_REFRESH_TOKEN`、`BOT_ADMIN`、`EDIT_MESSAGE_TTL_SECONDS`、`RUST_LOG`、`WEBHOOK*`、`TWITTER_AUTH_TOKEN`(可选)。
|
||||
|
||||
NSFW 推文:公开的 syndication 接口不返回敏感内容。设置 `TWITTER_AUTH_TOKEN`(登录 x.com 后浏览器 Cookie 里的 `auth_token` 值)后,bot 会仅在遇到 NSFW 推文时以登录态获取媒体;未设置则提示无媒体。
|
||||
|
||||
### Webhook 部署(需要反向代理)
|
||||
|
||||
`docker-compose.yml.example` 内置了 [nginx-proxy](https://github.com/nginx-proxy/nginx-proxy) + [acme-companion](https://github.com/nginx-proxy/acme-companion) 反向代理编排,按部署环境二选一:
|
||||
|
||||
**有域名**
|
||||
1. DNS A 记录指向服务器
|
||||
2. compose 里设 `VIRTUAL_HOST`、`LETSENCRYPT_HOST` 为域名,`WEBHOOK_URL` 设为 `https://域名/`
|
||||
3. 证书自动签发与续期,无需手动处理
|
||||
|
||||
**只有 IP**
|
||||
1. 生成自签证书(PEM 格式,见第 3 步):
|
||||
`openssl req -x509 -newkey rsa:2048 -nodes -days 365 -keyout nginx-certs/default.key -out nginx-certs/default.crt`
|
||||
2. compose 里 nginx-proxy 设 `DEFAULT_HOST`,bot 设 `WEBHOOK_CERT: './cert/cert.pem'`(须与代理所服务的为同一张证书)
|
||||
3. 证书必须是 PEM 编码(ASCII BASE64,以 `-----BEGIN CERTIFICATE-----` 开头)—— Telegram 只接受该格式;若现有证书是 DER 二进制,转换:
|
||||
`openssl x509 -in cert.der -inform DER -out cert.pem -outform PEM`
|
||||
(私钥同理:`openssl rsa -in key.der -inform DER -out key.pem -outform PEM`)
|
||||
|
||||
Telegram 只接受 443/80/88/8443 端口。
|
||||
|
||||
<details>
|
||||
<summary>环境变量说明</summary>
|
||||
|
||||
| 变量 | 说明 |
|
||||
|---|---|
|
||||
| `TELOXIDE_TOKEN` | Bot token(必填) |
|
||||
| `PIXIV_REFRESH_TOKEN` | Pixiv 刷新令牌;未设置则禁用 Pixiv |
|
||||
| `BOT_ADMIN` | 管理员聊天 ID,逗号分隔;接收启动/停止通知 |
|
||||
| `EDIT_MESSAGE_TTL_SECONDS` | 转发前编辑记录过期秒数,默认 86400 |
|
||||
| `RUST_LOG` | 日志级别 |
|
||||
| `LOCAL_USER_ID` | 容器内运行用户 UID,默认 9001 |
|
||||
| `VIRTUAL_HOST` | 对外域名或 IP,nginx-proxy 按此路由 |
|
||||
| `VIRTUAL_PORT` | bot 容器内监听端口,nginx-proxy 的转发目标 |
|
||||
| `LETSENCRYPT_HOST` | 设为域名时由 acme-companion 自动签发/续期证书 |
|
||||
| `DEFAULT_HOST` | nginx-proxy 将未知 Host 的请求路由到该 vhost(IP 访问时需要) |
|
||||
| `DEFAULT_EMAIL` | acme-companion 证书通知邮箱 |
|
||||
| `WEBHOOK` | `true` 启用 webhook 模式(默认轮询) |
|
||||
| `WEBHOOK_LISTEN` / `WEBHOOK_PORT` | bot 容器内监听地址/端口 |
|
||||
| `WEBHOOK_URL` | 对外公网 HTTPS 地址(`https://域名/`) |
|
||||
| `WEBHOOK_CERT` | 自签证书路径(仅 IP 路径需要,须为 PEM 且与代理所服务的一致) |
|
||||
| `WEBHOOK_SECRET_TOKEN` | 更新校验令牌(`X-Telegram-Bot-Api-Secret-Token`) |
|
||||
|
||||
</details>
|
||||
|
||||
## 命令
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "x-media"
|
||||
version = "1.0.0"
|
||||
version = "1.0.5"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
@@ -13,6 +13,7 @@ url = "2.5.2"
|
||||
bytes = "1"
|
||||
zip = "2"
|
||||
tempfile = "3"
|
||||
rand = "0.8"
|
||||
log = "0.4"
|
||||
tokio = { version = "1.40", features = ["time"] }
|
||||
|
||||
|
||||
@@ -14,6 +14,24 @@ impl Media {
|
||||
Media::Animated { thumbnail_url, .. } => Some(thumbnail_url),
|
||||
}
|
||||
}
|
||||
|
||||
/// A smaller variant of this media's file (used as the fallback when the
|
||||
/// primary URL or upload exceeds Telegram's size limits). None when no
|
||||
/// smaller variant exists (videos, animated gifs).
|
||||
pub fn smaller_url(&self) -> Option<&str> {
|
||||
match self {
|
||||
Media::Illustration {
|
||||
url,
|
||||
fallback_url,
|
||||
thumbnail_url,
|
||||
..
|
||||
} => fallback_url
|
||||
.as_deref()
|
||||
.or(thumbnail_url.as_deref())
|
||||
.filter(|smaller| *smaller != url),
|
||||
Media::Video { .. } | Media::Animated { .. } => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
|
||||
@@ -89,6 +89,9 @@ pub enum FetchError {
|
||||
Pixiv(PixivError),
|
||||
NotFound,
|
||||
Blocked,
|
||||
/// The post exists but its content is withheld (twitter NSFW /
|
||||
/// age-restricted tweets come back as an empty `{}` from syndication).
|
||||
Sensitive,
|
||||
}
|
||||
|
||||
impl fmt::Display for FetchError {
|
||||
@@ -99,6 +102,7 @@ impl fmt::Display for FetchError {
|
||||
FetchError::Pixiv(e) => write!(f, "pixiv error: {e}"),
|
||||
FetchError::NotFound => write!(f, "not found"),
|
||||
FetchError::Blocked => write!(f, "blocked"),
|
||||
FetchError::Sensitive => write!(f, "content withheld (sensitive)"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -109,7 +113,7 @@ impl std::error::Error for FetchError {
|
||||
FetchError::Http(e) => Some(e),
|
||||
FetchError::Json(e) => Some(e),
|
||||
FetchError::Pixiv(e) => Some(e),
|
||||
FetchError::NotFound | FetchError::Blocked => None,
|
||||
FetchError::NotFound | FetchError::Blocked | FetchError::Sensitive => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -195,6 +199,19 @@ async fn fetch_once(url: &str) -> Result<Option<Fetched>, FetchError> {
|
||||
/// fetch of a media URL is blocked (hotlink protection), the bot downloads
|
||||
/// the file itself and uploads it via multipart. Site-appropriate headers:
|
||||
/// pixiv image hosts need the `Referer` header.
|
||||
/// Returns the Content-Length of a media URL, or `None` when the server does
|
||||
/// not report one. Used to check whether a file fits Telegram's size limits
|
||||
/// before downloading/uploading it.
|
||||
pub async fn media_size(url: &str) -> Result<Option<u64>, FetchError> {
|
||||
let mut request = CLIENT.get(url);
|
||||
let lower = url.to_ascii_lowercase();
|
||||
if lower.contains("pximg.net") {
|
||||
request = request.header("Referer", "https://www.pixiv.net/");
|
||||
}
|
||||
let response = request.send().await?;
|
||||
Ok(response.content_length())
|
||||
}
|
||||
|
||||
pub async fn download_media(url: &str) -> Result<bytes::Bytes, FetchError> {
|
||||
let mut request = CLIENT.get(url);
|
||||
let lower = url.to_ascii_lowercase();
|
||||
|
||||
@@ -0,0 +1,376 @@
|
||||
//! Authenticated fallback for tweets the public syndication endpoint refuses
|
||||
//! to serve (NSFW / age-restricted tweets come back as an empty `{}`).
|
||||
//!
|
||||
//! Mirrors nazurin's web API client ([`web.py`]) and is used *only* when
|
||||
//! syndication reports [`FetchError::Sensitive`]: the private GraphQL
|
||||
//! `TweetDetail` endpoint, authenticated with a browser session cookie from
|
||||
//! `TWITTER_AUTH_TOKEN` (the `auth_token` cookie value of a logged-in x.com
|
||||
//! session). A fresh random `ct0` is generated per call; X checks that the
|
||||
//! `x-csrf-token` header matches the cookie, not that it issued the value.
|
||||
//!
|
||||
//! [`web.py`]: https://github.com/y-young/nazurin/blob/master/nazurin/sites/twitter/api/web.py
|
||||
//!
|
||||
//! # Caveats
|
||||
//! - X rotates the GraphQL query id when it rolls the web app; if requests
|
||||
//! start failing, update [`TWEET_DETAIL_QUERY_ID`]. Fresh references from
|
||||
//! the actively maintained FxEmbed/FxEmbed: TweetDetail
|
||||
//! `R9IzzyzQBV87-DOWpcvDmw`, TweetResultByRestId `f2sagi1jweVHFkTUIHzmMQ`
|
||||
//! (the latter is anonymous and surfaces NSFW tweets as
|
||||
//! `reason: NsfwLoggedOut`).
|
||||
//! - `x-client-transaction-id` is only required for `SearchTimeline`
|
||||
//! (verified against FxEmbed's `proxy/allowlist.ts`) — TweetDetail works
|
||||
//! without it; no need for the nazurin home-page/JS-bundle derivation.
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::site::FetchError;
|
||||
|
||||
use super::interface::Tweet;
|
||||
|
||||
/// `auth_token` cookie of a logged-in x.com session; enables the fallback.
|
||||
/// Trimmed: a CRLF `.env` (Windows) leaves a trailing `\r` on the value,
|
||||
/// which would make the Cookie header invalid.
|
||||
static AUTH_TOKEN: LazyLock<Option<String>> = LazyLock::new(|| {
|
||||
std::env::var("TWITTER_AUTH_TOKEN")
|
||||
.ok()
|
||||
.map(|s| s.trim().to_string())
|
||||
.filter(|s| !s.is_empty())
|
||||
});
|
||||
|
||||
/// Public "logged in" client token used by the x.com web app.
|
||||
const LOGGED_IN_BEARER: &str =
|
||||
"Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA";
|
||||
|
||||
/// `TweetDetail` query id (from nazurin; still valid as of 2026-08,
|
||||
/// corroborated by the current FxEmbed build — see module caveats).
|
||||
const TWEET_DETAIL_QUERY_ID: &str = "_8aYOgEDz35BrBcBal1-_w";
|
||||
|
||||
fn variables(id: &str) -> Value {
|
||||
json!({
|
||||
"focalTweetId": id,
|
||||
"with_rux_injections": false,
|
||||
"includePromotedContent": false,
|
||||
"withCommunity": true,
|
||||
"withQuickPromoteEligibilityTweetFields": false,
|
||||
"withBirdwatchNotes": false,
|
||||
"withVoice": true,
|
||||
})
|
||||
}
|
||||
|
||||
fn features() -> Value {
|
||||
json!({
|
||||
"rweb_video_screen_enabled": false,
|
||||
"profile_label_improvements_pcf_label_in_post_enabled": true,
|
||||
"rweb_tipjar_consumption_enabled": true,
|
||||
"verified_phone_label_enabled": false,
|
||||
"creator_subscriptions_tweet_preview_api_enabled": true,
|
||||
"responsive_web_graphql_timeline_navigation_enabled": true,
|
||||
"responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
|
||||
"premium_content_api_read_enabled": false,
|
||||
"communities_web_enable_tweet_community_results_fetch": true,
|
||||
"c9s_tweet_anatomy_moderator_badge_enabled": true,
|
||||
"responsive_web_grok_analyze_button_fetch_trends_enabled": false,
|
||||
"responsive_web_grok_analyze_post_followups_enabled": true,
|
||||
"responsive_web_jetfuel_frame": false,
|
||||
"responsive_web_grok_share_attachment_enabled": true,
|
||||
"articles_preview_enabled": true,
|
||||
"responsive_web_edit_tweet_api_enabled": true,
|
||||
"graphql_is_translatable_rweb_tweet_is_translatable_enabled": true,
|
||||
"view_counts_everywhere_api_enabled": true,
|
||||
"longform_notetweets_consumption_enabled": true,
|
||||
"responsive_web_twitter_article_tweet_consumption_enabled": true,
|
||||
"tweet_awards_web_tipping_enabled": false,
|
||||
"responsive_web_grok_show_grok_translated_post": false,
|
||||
"responsive_web_grok_analysis_button_from_backend": true,
|
||||
"creator_subscriptions_quote_tweet_preview_enabled": false,
|
||||
"freedom_of_speech_not_reach_fetch_enabled": true,
|
||||
"standardized_nudges_misinfo": true,
|
||||
"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true,
|
||||
"longform_notetweets_rich_text_read_enabled": true,
|
||||
"longform_notetweets_inline_media_enabled": true,
|
||||
"responsive_web_grok_image_annotation_enabled": true,
|
||||
"responsive_web_enhance_cards_enabled": false,
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether the authenticated fallback is available.
|
||||
pub fn enabled() -> bool {
|
||||
AUTH_TOKEN.is_some()
|
||||
}
|
||||
|
||||
/// Fetches a tweet as the logged-in user via the private GraphQL API.
|
||||
/// Returns the syndication-shaped [`Tweet`] (media included for NSFW posts).
|
||||
pub async fn fetch(id: &str) -> Result<Tweet, FetchError> {
|
||||
let token = AUTH_TOKEN
|
||||
.as_deref()
|
||||
.ok_or(FetchError::Sensitive)?;
|
||||
// 16 random bytes as 32 hex chars: X rejects ct0 values of any other
|
||||
// length with 403 code 353 ("matching csrf cookie and header").
|
||||
let ct0: String = (0..16)
|
||||
.map(|_| format!("{:02x}", rand::random::<u8>()))
|
||||
.collect();
|
||||
|
||||
let response = crate::site::CLIENT
|
||||
.get(format!(
|
||||
"https://x.com/i/api/graphql/{TWEET_DETAIL_QUERY_ID}/TweetDetail"
|
||||
))
|
||||
.query(&[
|
||||
("variables", variables(id).to_string()),
|
||||
("features", features().to_string()),
|
||||
])
|
||||
.header("authorization", LOGGED_IN_BEARER)
|
||||
.header("x-csrf-token", &ct0)
|
||||
.header("x-twitter-auth-type", "OAuth2Session")
|
||||
.header("cookie", format!("auth_token={token}; ct0={ct0}"))
|
||||
.header("x-twitter-client-language", "en")
|
||||
.header("x-twitter-active-user", "yes")
|
||||
.header("referer", "https://x.com/")
|
||||
.send()
|
||||
.await?;
|
||||
if !response.status().is_success() {
|
||||
log::warn!("twitter auth fetch {id}: HTTP {}", response.status());
|
||||
return Err(FetchError::NotFound);
|
||||
}
|
||||
let text = response.text().await?;
|
||||
let json: Value = serde_json::from_str(&text)?;
|
||||
let result = parse_tweet_result(&json, id)?;
|
||||
let syndication_shape = to_syndication_shape(&result)
|
||||
.ok_or_else(|| FetchError::Json(serde_json::Error::io(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"missing tweet fields in GraphQL response",
|
||||
))))?;
|
||||
Tweet::from_syndication_json(&syndication_shape.to_string()).map_err(FetchError::Json)
|
||||
}
|
||||
|
||||
/// Locates the tweet for `id` in a `TweetDetail` response and unwraps
|
||||
/// visibility wrappers / retweets, mirroring nazurin's `_process_response`.
|
||||
fn parse_tweet_result(json: &Value, id: &str) -> Result<Value, FetchError> {
|
||||
if let Some(errors) = json.get("errors").and_then(|e| e.as_array()) {
|
||||
let messages: Vec<&str> = errors
|
||||
.iter()
|
||||
.filter_map(|e| e.get("message").and_then(|m| m.as_str()))
|
||||
.collect();
|
||||
log::warn!("twitter auth fetch {id} failed: {}", messages.join("; "));
|
||||
return Err(FetchError::NotFound);
|
||||
}
|
||||
|
||||
let instructions = json
|
||||
.pointer("/data/threaded_conversation_with_injections_v2/instructions")
|
||||
.and_then(|v| v.as_array())
|
||||
.ok_or(FetchError::NotFound)?;
|
||||
for instruction in instructions {
|
||||
if instruction.get("type").and_then(|t| t.as_str()) != Some("TimelineAddEntries") {
|
||||
continue;
|
||||
}
|
||||
let entries = instruction
|
||||
.get("entries")
|
||||
.and_then(|e| e.as_array())
|
||||
.ok_or(FetchError::NotFound)?;
|
||||
let wanted = format!("tweet-{id}");
|
||||
for entry in entries {
|
||||
if entry.get("entryId").and_then(|i| i.as_str()) == Some(wanted.as_str()) {
|
||||
let result = entry
|
||||
.pointer("/content/itemContent/tweet_results/result")
|
||||
.ok_or(FetchError::NotFound)?;
|
||||
return normalize_tweet_result(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(FetchError::NotFound)
|
||||
}
|
||||
|
||||
/// Unwraps TweetTombstone/TweetUnavailable errors, the
|
||||
/// TweetWithVisibilityResults wrapper and retweets, returning the
|
||||
/// `{core, legacy, ...}` tweet object.
|
||||
fn normalize_tweet_result(result: &Value) -> Result<Value, FetchError> {
|
||||
match result.get("__typename").and_then(|t| t.as_str()) {
|
||||
Some("TweetTombstone") => {
|
||||
let text = result
|
||||
.pointer("/tombstone/text/text")
|
||||
.and_then(|t| t.as_str())
|
||||
.unwrap_or("tweet is unavailable");
|
||||
log::warn!("twitter auth fetch: tombstone: {text}");
|
||||
return Err(FetchError::NotFound);
|
||||
}
|
||||
Some("TweetUnavailable") => {
|
||||
let reason = result
|
||||
.get("reason")
|
||||
.and_then(|r| r.as_str())
|
||||
.unwrap_or("unknown");
|
||||
log::warn!("twitter auth fetch: tweet unavailable: {reason}");
|
||||
return Err(FetchError::NotFound);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
// TweetWithVisibilityResults (e.g. limited replies) nests the real tweet.
|
||||
let tweet = result.get("tweet").unwrap_or(result);
|
||||
// A retweet's media lives on the original tweet.
|
||||
if let Some(original) = tweet.pointer("/legacy/retweeted_status_result/result") {
|
||||
return Ok(original.clone());
|
||||
}
|
||||
Ok(tweet.clone())
|
||||
}
|
||||
|
||||
/// Maps a GraphQL `{core, legacy, ...}` tweet onto the syndication JSON
|
||||
/// shape [`Tweet::from_syndication_json`] parses, so the existing text /
|
||||
/// media handling (t.co expansion, `name=orig`, mp4 variant) is reused.
|
||||
fn to_syndication_shape(tweet: &Value) -> Option<Value> {
|
||||
let legacy = tweet.get("legacy")?;
|
||||
let user = tweet.pointer("/core/user_results/result/legacy")?;
|
||||
Some(json!({
|
||||
"id_str": legacy.get("id_str"),
|
||||
"text": legacy.get("full_text"),
|
||||
"user": {
|
||||
"name": user.get("name"),
|
||||
"screen_name": user.get("screen_name"),
|
||||
},
|
||||
"possibly_sensitive": legacy.get("possibly_sensitive"),
|
||||
"display_text_range": legacy.get("display_text_range"),
|
||||
"entities": legacy.get("entities"),
|
||||
"mediaDetails": legacy.pointer("/extended_entities/media"),
|
||||
}))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn tweet_result() -> Value {
|
||||
json!({
|
||||
"__typename": "Tweet",
|
||||
"core": {
|
||||
"user_results": {
|
||||
"result": {
|
||||
"legacy": { "name": "Display Name", "screen_name": "nsfw_author" }
|
||||
}
|
||||
}
|
||||
},
|
||||
"legacy": {
|
||||
"id_str": "2083868672721039569",
|
||||
"full_text": "nsfw content https://t.co/abc123",
|
||||
"display_text_range": [0, 12],
|
||||
"possibly_sensitive": true,
|
||||
"entities": {
|
||||
"urls": [
|
||||
{ "url": "https://t.co/abc123", "expanded_url": "https://example.com/x" }
|
||||
]
|
||||
},
|
||||
"extended_entities": {
|
||||
"media": [
|
||||
{
|
||||
"type": "photo",
|
||||
"media_url_https": "https://pbs.twimg.com/media/nsfw.jpg",
|
||||
"original_info": { "width": 1200, "height": 800 }
|
||||
},
|
||||
{
|
||||
"type": "video",
|
||||
"media_url_https": "https://pbs.twimg.com/thumb.jpg",
|
||||
"video_info": {
|
||||
"variants": [
|
||||
{ "content_type": "application/x-mpegURL", "url": "https://x.com/pl.m3u8" },
|
||||
{ "content_type": "video/mp4", "url": "https://video.twimg.com/nsfw.mp4" }
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn conversation(tweet: Value) -> Value {
|
||||
json!({
|
||||
"data": {
|
||||
"threaded_conversation_with_injections_v2": {
|
||||
"instructions": [
|
||||
{ "type": "TimelineAddEntries", "entries": [
|
||||
{ "entryId": "tweet-2083868672721039569",
|
||||
"content": { "itemContent": { "tweet_results": { "result": tweet } } } }
|
||||
]}
|
||||
]
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_graphql_tweet_into_fetched() {
|
||||
let json = conversation(tweet_result());
|
||||
let result = parse_tweet_result(&json, "2083868672721039569").unwrap();
|
||||
let shape = to_syndication_shape(&result).unwrap();
|
||||
let tweet = Tweet::from_syndication_json(&shape.to_string()).unwrap();
|
||||
let fetched: crate::site::Fetched = tweet.into();
|
||||
|
||||
assert!(fetched.sensitive);
|
||||
assert_eq!(fetched.media.len(), 2);
|
||||
match &fetched.media[0] {
|
||||
crate::media::Media::Illustration { url, .. } => {
|
||||
assert_eq!(url, "https://pbs.twimg.com/media/nsfw.jpg?name=orig");
|
||||
}
|
||||
other => panic!("expected illustration, got {other:?}"),
|
||||
}
|
||||
match &fetched.media[1] {
|
||||
crate::media::Media::Video { url, .. } => {
|
||||
assert_eq!(url, "https://video.twimg.com/nsfw.mp4");
|
||||
}
|
||||
other => panic!("expected video, got {other:?}"),
|
||||
}
|
||||
assert_eq!(fetched.source_url, "https://x.com/nsfw_author/status/2083868672721039569");
|
||||
// display_text_range cuts the trailing t.co link.
|
||||
assert_eq!(fetched.title, "nsfw content");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unwraps_retweet_to_original() {
|
||||
let original = tweet_result();
|
||||
let mut rt = tweet_result();
|
||||
rt["legacy"]["retweeted_status_result"] = json!({ "result": original });
|
||||
let json = conversation(rt);
|
||||
let result = parse_tweet_result(&json, "2083868672721039569").unwrap();
|
||||
assert!(result.pointer("/legacy/retweeted_status_result").is_none());
|
||||
assert_eq!(result.pointer("/legacy/id_str").unwrap(), "2083868672721039569");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_response_maps_to_not_found() {
|
||||
let json = json!({ "errors": [{ "message": "NsfwLoggedOut" }] });
|
||||
assert!(matches!(
|
||||
parse_tweet_result(&json, "1"),
|
||||
Err(FetchError::NotFound)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_entry_maps_to_not_found() {
|
||||
let json = conversation(json!({ "__typename": "Tweet" }));
|
||||
assert!(matches!(
|
||||
parse_tweet_result(&json, "999"),
|
||||
Err(FetchError::NotFound)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tombstone_maps_to_not_found() {
|
||||
let tombstone = json!({
|
||||
"__typename": "TweetTombstone",
|
||||
"tombstone": { "text": { "text": "Age-restricted adult content" } }
|
||||
});
|
||||
let json = conversation(tombstone);
|
||||
assert!(matches!(
|
||||
parse_tweet_result(&json, "2083868672721039569"),
|
||||
Err(FetchError::NotFound)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn visibility_wrapper_unwraps() {
|
||||
let inner = tweet_result();
|
||||
let wrapped = json!({ "__typename": "TweetWithVisibilityResults", "tweet": inner });
|
||||
let json = conversation(wrapped);
|
||||
let result = parse_tweet_result(&json, "2083868672721039569").unwrap();
|
||||
assert_eq!(result.get("__typename").unwrap(), "Tweet");
|
||||
}
|
||||
}
|
||||
@@ -19,7 +19,44 @@ pub async fn fetch_from_url(url: &str) -> Result<Fetched, FetchError> {
|
||||
.and_then(|caps| caps.get(1))
|
||||
.map(|m| m.as_str())
|
||||
.ok_or(FetchError::NotFound)?;
|
||||
Ok(fetch(id).await?.into())
|
||||
match fetch(id).await {
|
||||
Ok(tweet) => Ok(tweet.into()),
|
||||
// Syndication withholds NSFW/age-restricted tweets (empty `{}`).
|
||||
// Retry as the logged-in user when TWITTER_AUTH_TOKEN is set;
|
||||
// otherwise degrade to an empty result (the bot replies
|
||||
// "No media found").
|
||||
Err(FetchError::Sensitive) => {
|
||||
if super::auth::enabled() {
|
||||
match super::auth::fetch(id).await {
|
||||
Ok(tweet) => Ok(tweet.into()),
|
||||
Err(e) => {
|
||||
log::warn!("twitter auth fallback failed for {id}: {e}");
|
||||
Ok(empty_fetched(url))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log::info!(
|
||||
"tweet {id} is sensitive; set TWITTER_AUTH_TOKEN to fetch NSFW media"
|
||||
);
|
||||
Ok(empty_fetched(url))
|
||||
}
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
/// A Fetched with no media for withheld tweets: the bot replies
|
||||
/// "No media found" and moves on instead of erroring.
|
||||
fn empty_fetched(url: &str) -> Fetched {
|
||||
Fetched {
|
||||
source_url: url.to_string(),
|
||||
caption: url.to_string(),
|
||||
title: String::new(),
|
||||
media: vec![],
|
||||
sensitive: true,
|
||||
render_data: None,
|
||||
_keep_alive: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetches a tweet from the syndication endpoint. Deleted/blocked tweets
|
||||
@@ -44,6 +81,15 @@ pub async fn fetch(id: &str) -> Result<Tweet, FetchError> {
|
||||
{
|
||||
return Err(FetchError::NotFound);
|
||||
}
|
||||
// NSFW / age-restricted tweets exist but are served as an empty `{}` —
|
||||
// they surface as FetchError::Sensitive so the caller can retry as a
|
||||
// logged-in user.
|
||||
if serde_json::from_str::<serde_json::Value>(&text)
|
||||
.map(|v| v.get("id_str").is_none())
|
||||
.unwrap_or(false)
|
||||
{
|
||||
return Err(FetchError::Sensitive);
|
||||
}
|
||||
Ok(Tweet::from_syndication_json(&text).map_err(FetchError::Json)?)
|
||||
}
|
||||
|
||||
@@ -127,9 +173,11 @@ impl Tweet {
|
||||
match item.media_type.as_str() {
|
||||
"photo" => media.push(Media::Illustration {
|
||||
title: None,
|
||||
url: item.media_url_https,
|
||||
url: original_twimg_url(&item.media_url_https),
|
||||
thumbnail_url: None,
|
||||
fallback_url: None,
|
||||
// The param-less base URL is a reduced-size variant;
|
||||
// used as the fallback when the original is too large.
|
||||
fallback_url: Some(item.media_url_https.clone()),
|
||||
}),
|
||||
"video" => media.push(Media::Video {
|
||||
title: None,
|
||||
@@ -157,21 +205,17 @@ impl Tweet {
|
||||
}
|
||||
|
||||
/// The raw syndication `text` ends with the appended media short link
|
||||
/// (" https://t.co/wmI8McgXul"). `display_text_range` (UTF-16 indices) marks
|
||||
/// the visible text; a regex strips any remaining trailing t.co link when the
|
||||
/// range is absent or a tweet ends in a URL short link.
|
||||
/// (" https://t.co/wmI8McgXul"). `display_text_range` marks the visible text;
|
||||
/// a regex strips any remaining trailing t.co link when the range is absent
|
||||
/// or a tweet ends in a URL short link.
|
||||
///
|
||||
/// X reports these indices in Unicode **code points**, not UTF-16 units
|
||||
/// (verified against GraphQL responses containing emoji: cutting an emoji
|
||||
/// tweet by UTF-16 units silently drops the character after the emoji).
|
||||
fn strip_trailing_short_links(text: &str, display_text_range: Option<[usize; 2]>) -> String {
|
||||
let mut out = match display_text_range {
|
||||
Some([start, end]) if start < end => {
|
||||
let units: Vec<u16> = text
|
||||
.encode_utf16()
|
||||
.skip(start)
|
||||
.take(end - start)
|
||||
.collect();
|
||||
// Drop the replacement char that a surrogate cut at the boundary
|
||||
// would produce (the range end is a valid UTF-16 boundary in
|
||||
// practice, so this is just a safety net).
|
||||
String::from_utf16_lossy(&units).replace('\u{FFFD}', "")
|
||||
text.chars().skip(start).take(end - start).collect()
|
||||
}
|
||||
_ => text.to_string(),
|
||||
};
|
||||
@@ -198,6 +242,20 @@ fn expand_links(text: &str, urls: &[model::SyndicationEntityUrl]) -> String {
|
||||
out
|
||||
}
|
||||
|
||||
/// pbs.twimg.com serves a reduced default size without size params; `name=orig`
|
||||
/// returns the original file (fxtwitter used to hand out the original
|
||||
/// directly, the syndication API does not). Non-twimg URLs pass through
|
||||
/// unchanged.
|
||||
fn original_twimg_url(url: &str) -> String {
|
||||
if url.starts_with("https://pbs.twimg.com/")
|
||||
&& (url.ends_with(".jpg") || url.ends_with(".png"))
|
||||
{
|
||||
format!("{url}?name=orig")
|
||||
} else {
|
||||
url.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
fn mp4_variant(item: &model::SyndicationMedia) -> String {
|
||||
item.video_info
|
||||
.as_ref()
|
||||
@@ -302,6 +360,16 @@ mod tests {
|
||||
assert_eq!(fetched.title, "a & b <c>");
|
||||
assert!(fetched.sensitive);
|
||||
assert_eq!(fetched.media.len(), 2);
|
||||
match &fetched.media[0] {
|
||||
Media::Illustration { url, .. } => {
|
||||
// Photo URL is rewritten to request the original file.
|
||||
assert_eq!(
|
||||
url,
|
||||
"https://pbs.twimg.com/media/photo.jpg?name=orig"
|
||||
);
|
||||
}
|
||||
other => panic!("expected illustration, got {other:?}"),
|
||||
}
|
||||
match &fetched.media[1] {
|
||||
Media::Video { url, thumbnail_url, .. } => {
|
||||
assert_eq!(url, "https://video.twimg.com/v.mp4");
|
||||
@@ -435,6 +503,27 @@ mod tests {
|
||||
assert_eq!(tweet.text, text, "full text kept intact");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn original_twimg_url_rewrites_photo_urls() {
|
||||
assert_eq!(
|
||||
original_twimg_url("https://pbs.twimg.com/media/C_UdnvPUwAE3Dnn.jpg"),
|
||||
"https://pbs.twimg.com/media/C_UdnvPUwAE3Dnn.jpg?name=orig"
|
||||
);
|
||||
assert_eq!(
|
||||
original_twimg_url("https://pbs.twimg.com/media/abc.png"),
|
||||
"https://pbs.twimg.com/media/abc.png?name=orig"
|
||||
);
|
||||
// Non-twimg URLs (videos, animated gifs) pass through unchanged.
|
||||
assert_eq!(
|
||||
original_twimg_url("https://video.twimg.com/v.mp4"),
|
||||
"https://video.twimg.com/v.mp4"
|
||||
);
|
||||
assert_eq!(
|
||||
original_twimg_url("https://pbs.twimg.com/media/abc.webp"),
|
||||
"https://pbs.twimg.com/media/abc.webp"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn syndication_token_matches_js_formula() {
|
||||
// JS: ((861627479294746624 / 1e15) * PI).toString(36) == "236.vrsocvda"
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
mod auth;
|
||||
mod interface;
|
||||
mod model;
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ pub struct SyndicationTweet {
|
||||
#[serde(default)]
|
||||
pub possibly_sensitive: Option<bool>,
|
||||
/// Visible-text span; the raw `text` field has the appended media short
|
||||
/// link after it. Indices are UTF-16 code units.
|
||||
/// link after it. Indices are Unicode code points (not UTF-16 units).
|
||||
#[serde(default, rename = "display_text_range")]
|
||||
pub display_text_range: Option<[usize; 2]>,
|
||||
#[serde(default)]
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "xmedia-bot"
|
||||
version = "1.0.0"
|
||||
version = "1.0.5"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
|
||||
@@ -41,8 +41,14 @@ impl Config {
|
||||
let webhook_url = env::var("WEBHOOK_URL").ok().and_then(|s| s.parse().ok());
|
||||
let webhook_listen = env::var("WEBHOOK_LISTEN").ok().and_then(|s| s.parse().ok());
|
||||
let webhook_port = env::var("WEBHOOK_PORT").ok().and_then(|s| s.parse().ok());
|
||||
let webhook_cert = env::var("WEBHOOK_CERT").ok();
|
||||
let webhook_secret_token = env::var("WEBHOOK_SECRET_TOKEN").ok();
|
||||
// Empty strings count as unset (e.g. `-e WEBHOOK_CERT=` to disable a
|
||||
// value that would otherwise come from `.env`).
|
||||
let webhook_cert = env::var("WEBHOOK_CERT")
|
||||
.ok()
|
||||
.filter(|s| !s.is_empty());
|
||||
let webhook_secret_token = env::var("WEBHOOK_SECRET_TOKEN")
|
||||
.ok()
|
||||
.filter(|s| !s.is_empty());
|
||||
|
||||
Config {
|
||||
admin_ids,
|
||||
|
||||
@@ -322,22 +322,26 @@ fn thumbnail_for(media: &Media) -> Option<String> {
|
||||
}
|
||||
|
||||
fn media_to_payload(media: &Media, sensitive: bool) -> MediaItemPayload {
|
||||
let fallback_url = media.smaller_url().map(str::to_string);
|
||||
match media {
|
||||
// A gif inside a group becomes a video item; a lone gif takes the
|
||||
// animation path (see url_media).
|
||||
Media::Illustration { .. } => MediaItemPayload::Photo {
|
||||
media: media.url().to_string(),
|
||||
has_spoiler: sensitive,
|
||||
fallback_url,
|
||||
},
|
||||
Media::Video { .. } => MediaItemPayload::Video {
|
||||
media: media.url().to_string(),
|
||||
has_spoiler: sensitive,
|
||||
thumbnail: thumbnail_for(media),
|
||||
fallback_url,
|
||||
},
|
||||
Media::Animated { .. } => MediaItemPayload::Video {
|
||||
media: media.url().to_string(),
|
||||
has_spoiler: sensitive,
|
||||
thumbnail: thumbnail_for(media),
|
||||
fallback_url,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -502,11 +506,19 @@ pub async fn inline_query_handler(bot: Bot, query: InlineQuery) -> Result<(), Re
|
||||
.unwrap_or_else(|| url.clone());
|
||||
let caption = fetched.caption.clone();
|
||||
let result = match media {
|
||||
Media::Illustration { .. } => InlineQueryResult::Photo(
|
||||
InlineQueryResultPhoto::new(id, url, thumbnail)
|
||||
.caption(caption)
|
||||
.parse_mode(ParseMode::Html),
|
||||
),
|
||||
Media::Illustration { .. } => {
|
||||
// Inline photo results have their own (smaller) size
|
||||
// cap; use the reduced variant when one exists.
|
||||
let photo_url = media
|
||||
.smaller_url()
|
||||
.and_then(|u| url::Url::parse(u).ok())
|
||||
.unwrap_or_else(|| url.clone());
|
||||
InlineQueryResult::Photo(
|
||||
InlineQueryResultPhoto::new(id, photo_url, thumbnail)
|
||||
.caption(caption)
|
||||
.parse_mode(ParseMode::Html),
|
||||
)
|
||||
}
|
||||
Media::Video { .. } => InlineQueryResult::Video(
|
||||
InlineQueryResultVideo::new(
|
||||
id,
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
use dotenv::dotenv;
|
||||
use teloxide::dptree::endpoint;
|
||||
use teloxide::stop::StopToken;
|
||||
use teloxide::types::{ChatId, InputFile, MessageId};
|
||||
use teloxide::update_listeners::webhooks;
|
||||
use teloxide::update_listeners::{self, webhooks, UpdateListener};
|
||||
use teloxide::prelude::*;
|
||||
use tokio::sync::watch;
|
||||
use x_media::site;
|
||||
@@ -14,6 +15,24 @@ mod state;
|
||||
|
||||
use handlers::{CHAT_STORE, CONFIG, TASK_QUEUE};
|
||||
|
||||
/// Docker `stop` / `compose down` delivers SIGTERM, which teloxide's ctrlc
|
||||
/// handler (SIGINT only) never sees — without this the process would die
|
||||
/// before the graceful shutdown below (admin notice, queue drain). Stopping
|
||||
/// the token unwinds the dispatcher exactly like Ctrl+C does.
|
||||
#[cfg(unix)]
|
||||
fn spawn_sigterm_handler(stop_token: StopToken) {
|
||||
tokio::spawn(async move {
|
||||
let mut sigterm = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
|
||||
.expect("failed to install SIGTERM handler");
|
||||
sigterm.recv().await;
|
||||
log::info!("SIGTERM received, stopping the dispatcher");
|
||||
stop_token.stop();
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn spawn_sigterm_handler(_stop_token: StopToken) {}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
dotenv().ok();
|
||||
@@ -96,7 +115,8 @@ async fn main() {
|
||||
.webhook_url
|
||||
.clone()
|
||||
.expect("WEBHOOK_URL is not set");
|
||||
bot.set_webhook(url.clone()).await.unwrap();
|
||||
// `webhooks::axum` calls set_webhook itself (with the full options,
|
||||
// secret token included) — no explicit registration here.
|
||||
let listen = CONFIG.webhook_listen.expect("WEBHOOK_LISTEN is not set");
|
||||
let port = CONFIG.webhook_port.expect("WEBHOOK_PORT is not set");
|
||||
let mut options = webhooks::Options::new((listen, port).into(), url);
|
||||
@@ -107,20 +127,36 @@ async fn main() {
|
||||
options = options.secret_token(secret.clone());
|
||||
}
|
||||
|
||||
let mut listener = webhooks::axum(bot.clone(), options)
|
||||
.await
|
||||
.expect("Failed to create webhook listener");
|
||||
let stop_token = listener.stop_token();
|
||||
spawn_sigterm_handler(stop_token);
|
||||
|
||||
dispatcher
|
||||
.dispatch_with_listener(
|
||||
webhooks::axum(bot.clone(), options)
|
||||
.await
|
||||
.expect("Failed to create webhook listener"),
|
||||
listener,
|
||||
LoggingErrorHandler::with_custom_text("Error from update listener"),
|
||||
)
|
||||
.await;
|
||||
} else {
|
||||
log::info!("running in polling mode");
|
||||
dispatcher.dispatch().await;
|
||||
// Same listener `dispatch()` builds internally — using
|
||||
// `dispatch_with_listener` just exposes its stop token so SIGTERM can
|
||||
// unwind the dispatcher before the graceful shutdown below.
|
||||
let mut listener = update_listeners::polling_default(bot.clone()).await;
|
||||
let stop_token = listener.stop_token();
|
||||
spawn_sigterm_handler(stop_token);
|
||||
|
||||
dispatcher
|
||||
.dispatch_with_listener(
|
||||
listener,
|
||||
LoggingErrorHandler::with_custom_text("Error from update listener"),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
// Graceful stop (Ctrl+C): stop the sweep, notify the admin, drain the queue.
|
||||
// Graceful stop (Ctrl+C / SIGTERM): stop the sweep, notify the admin, drain the queue.
|
||||
log::info!("Stopping bot");
|
||||
let _ = stop_tx.send(true);
|
||||
if let Some(admin) = CONFIG.admin_ids.first() {
|
||||
|
||||
+206
-36
@@ -25,11 +25,17 @@ pub enum MediaItemPayload {
|
||||
Photo {
|
||||
media: String,
|
||||
has_spoiler: bool,
|
||||
/// Smaller variant used when the primary media exceeds Telegram's
|
||||
/// size limits.
|
||||
#[serde(default)]
|
||||
fallback_url: Option<String>,
|
||||
},
|
||||
Video {
|
||||
media: String,
|
||||
has_spoiler: bool,
|
||||
thumbnail: Option<String>,
|
||||
#[serde(default)]
|
||||
fallback_url: Option<String>,
|
||||
},
|
||||
Animation {
|
||||
media: String,
|
||||
@@ -37,6 +43,16 @@ pub enum MediaItemPayload {
|
||||
},
|
||||
}
|
||||
|
||||
impl MediaItemPayload {
|
||||
fn fallback_url(&self) -> Option<&str> {
|
||||
match self {
|
||||
MediaItemPayload::Photo { fallback_url, .. }
|
||||
| MediaItemPayload::Video { fallback_url, .. } => fallback_url.as_deref(),
|
||||
MediaItemPayload::Animation { .. } => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone, Debug)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum Task {
|
||||
@@ -74,7 +90,9 @@ pub enum Task {
|
||||
}
|
||||
|
||||
pub const MAX_MEDIA_GROUP: usize = 9;
|
||||
pub const MAX_UPLOAD_BYTES: u64 = 50 * 1024 * 1024; // Telegram Bot API upload cap
|
||||
/// Upload cap (bytes): files above this are not uploaded; the bot falls back
|
||||
/// to a smaller media URL instead.
|
||||
pub const MAX_UPLOAD_BYTES: u64 = 10 * 1024 * 1024; // 10485760
|
||||
|
||||
/// Splits media into batches of at most [`MAX_MEDIA_GROUP`] items.
|
||||
pub fn chunk_media_items<T: Clone>(items: Vec<T>) -> Vec<Vec<T>> {
|
||||
@@ -102,6 +120,18 @@ pub fn is_media_fetch_failure(e: &ApiError) -> bool {
|
||||
MARKERS.iter().any(|marker| description.contains(marker))
|
||||
}
|
||||
|
||||
/// Telegram reported the media file as too large (HTTP 413 on multipart
|
||||
/// upload, or a "too large" message for URL-fetched media). These errors are
|
||||
/// handled by the size-check fallback (use a smaller media URL), NOT by a
|
||||
/// queue retry.
|
||||
pub fn is_size_error(e: &ApiError) -> bool {
|
||||
if matches!(e, ApiError::RequestEntityTooLarge) {
|
||||
return true;
|
||||
}
|
||||
let description = e.to_string().to_lowercase();
|
||||
["too large", "too big"].iter().any(|marker| description.contains(marker))
|
||||
}
|
||||
|
||||
/// Task-free classification of a Telegram request error. The callers attach
|
||||
/// the (updated) task when building a [`SendError`].
|
||||
pub enum Classification {
|
||||
@@ -216,11 +246,13 @@ fn build_media_group(
|
||||
MediaItemPayload::Photo {
|
||||
media,
|
||||
has_spoiler,
|
||||
..
|
||||
} => photo_media(input_file_for(media)?, item_caption, *has_spoiler),
|
||||
MediaItemPayload::Video {
|
||||
media,
|
||||
has_spoiler,
|
||||
thumbnail,
|
||||
..
|
||||
} => {
|
||||
let mut video = video_media(input_file_for(media)?, item_caption, *has_spoiler);
|
||||
if let (Some(thumb), InputMedia::Video(v)) = (thumbnail, &mut video) {
|
||||
@@ -231,6 +263,7 @@ fn build_media_group(
|
||||
MediaItemPayload::Animation {
|
||||
media,
|
||||
has_spoiler,
|
||||
..
|
||||
} => animation_media(input_file_for(media)?, item_caption, *has_spoiler),
|
||||
})
|
||||
})
|
||||
@@ -258,6 +291,9 @@ fn sniff_ext(bytes: &[u8]) -> &'static str {
|
||||
enum FallbackError {
|
||||
Retryable { delay_seconds: f64 },
|
||||
Permanent { message: String },
|
||||
/// The downloaded file exceeds the upload cap; the caller falls back to
|
||||
/// the item's smaller URL.
|
||||
MediaTooLarge,
|
||||
}
|
||||
|
||||
/// Downloads one media item to a temp file (deleted on drop). Network errors
|
||||
@@ -282,9 +318,7 @@ async fn download_to_temp(item: &MediaItemPayload) -> Result<NamedTempFile, Fall
|
||||
}
|
||||
};
|
||||
if bytes.len() as u64 > MAX_UPLOAD_BYTES {
|
||||
return Err(FallbackError::Permanent {
|
||||
message: "media too large".into(),
|
||||
});
|
||||
return Err(FallbackError::MediaTooLarge);
|
||||
}
|
||||
let ext = sniff_ext(&bytes);
|
||||
let mut file = tempfile::Builder::new()
|
||||
@@ -302,7 +336,48 @@ async fn download_to_temp(item: &MediaItemPayload) -> Result<NamedTempFile, Fall
|
||||
Ok(file)
|
||||
}
|
||||
|
||||
/// Download-and-reupload fallback for one media batch.
|
||||
/// Builds the media group item from an uploaded file.
|
||||
fn media_from_file(
|
||||
item: &MediaItemPayload,
|
||||
path: std::path::PathBuf,
|
||||
caption: Option<&str>,
|
||||
) -> InputMedia {
|
||||
match item {
|
||||
MediaItemPayload::Photo { has_spoiler, .. } => {
|
||||
photo_media(InputFile::file(path), caption, *has_spoiler)
|
||||
}
|
||||
MediaItemPayload::Video { has_spoiler, .. } => {
|
||||
video_media(InputFile::file(path), caption, *has_spoiler)
|
||||
}
|
||||
MediaItemPayload::Animation { has_spoiler, .. } => {
|
||||
animation_media(InputFile::file(path), caption, *has_spoiler)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds the media group item from a (smaller) URL.
|
||||
fn media_from_url(
|
||||
item: &MediaItemPayload,
|
||||
url: &str,
|
||||
caption: Option<&str>,
|
||||
) -> Result<InputMedia, String> {
|
||||
Ok(match item {
|
||||
MediaItemPayload::Photo { has_spoiler, .. } => {
|
||||
photo_media(input_file_for(url)?, caption, *has_spoiler)
|
||||
}
|
||||
MediaItemPayload::Video { has_spoiler, .. } => {
|
||||
video_media(input_file_for(url)?, caption, *has_spoiler)
|
||||
}
|
||||
MediaItemPayload::Animation { has_spoiler, .. } => {
|
||||
animation_media(input_file_for(url)?, caption, *has_spoiler)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Download-and-reupload fallback for one media batch. Files over the upload
|
||||
/// cap are not downloaded/uploaded; the item falls back to its smaller URL
|
||||
/// (which Telegram fetches itself). Returns the fallback-error without the
|
||||
/// task attached; callers wrap it with the updated task state.
|
||||
async fn send_batch_via_upload(
|
||||
bot: &Bot,
|
||||
chat_id: i64,
|
||||
@@ -313,22 +388,51 @@ async fn send_batch_via_upload(
|
||||
let mut files = Vec::new();
|
||||
let mut items = Vec::new();
|
||||
for (i, item) in batch.iter().enumerate() {
|
||||
let file = download_to_temp(item).await?;
|
||||
let path = file.path().to_path_buf();
|
||||
let item_caption = if i == 0 { caption } else { None };
|
||||
let media = match item {
|
||||
MediaItemPayload::Photo { has_spoiler, .. } => {
|
||||
photo_media(InputFile::file(path), item_caption, *has_spoiler)
|
||||
// Size check before downloading/uploading: over the cap, use the
|
||||
// smaller URL instead of the file.
|
||||
let too_large = match x_media::site::media_size(item_url(item)).await {
|
||||
Ok(Some(size)) => size > MAX_UPLOAD_BYTES,
|
||||
_ => false,
|
||||
};
|
||||
let media = if too_large {
|
||||
match item.fallback_url() {
|
||||
Some(url) => match media_from_url(item, url, item_caption) {
|
||||
Ok(media) => media,
|
||||
Err(message) => {
|
||||
return Err(FallbackError::Permanent { message });
|
||||
}
|
||||
},
|
||||
None => {
|
||||
return Err(FallbackError::Permanent {
|
||||
message: "media too large".into(),
|
||||
});
|
||||
}
|
||||
}
|
||||
MediaItemPayload::Video { has_spoiler, .. } => {
|
||||
video_media(InputFile::file(path), item_caption, *has_spoiler)
|
||||
}
|
||||
MediaItemPayload::Animation { has_spoiler, .. } => {
|
||||
animation_media(InputFile::file(path), item_caption, *has_spoiler)
|
||||
} else {
|
||||
match download_to_temp(item).await {
|
||||
Ok(file) => {
|
||||
let path = file.path().to_path_buf();
|
||||
files.push(file);
|
||||
media_from_file(item, path, item_caption)
|
||||
}
|
||||
Err(FallbackError::MediaTooLarge) => match item.fallback_url() {
|
||||
Some(url) => match media_from_url(item, url, item_caption) {
|
||||
Ok(media) => media,
|
||||
Err(message) => {
|
||||
return Err(FallbackError::Permanent { message });
|
||||
}
|
||||
},
|
||||
None => {
|
||||
return Err(FallbackError::Permanent {
|
||||
message: "media too large".into(),
|
||||
});
|
||||
}
|
||||
},
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
};
|
||||
items.push(media);
|
||||
files.push(file);
|
||||
}
|
||||
let result = bot
|
||||
.send_media_group(ChatId(chat_id), items)
|
||||
@@ -422,7 +526,9 @@ pub async fn send_media_sequence(bot: &Bot, task: &Task) -> Result<Vec<i64>, Sen
|
||||
);
|
||||
sent.extend(messages.into_iter().map(|m| m.id.0 as i64));
|
||||
}
|
||||
Err(RequestError::Api(api)) if is_media_fetch_failure(&api) => {
|
||||
Err(RequestError::Api(api))
|
||||
if is_media_fetch_failure(&api) || is_size_error(&api) =>
|
||||
{
|
||||
log::info!(
|
||||
"Telegram could not fetch media for batch {idx} ({}), downloading and reuploading",
|
||||
batch
|
||||
@@ -444,6 +550,7 @@ pub async fn send_media_sequence(bot: &Bot, task: &Task) -> Result<Vec<i64>, Sen
|
||||
task: updated_sequence_task(task, idx, sent),
|
||||
});
|
||||
}
|
||||
Err(FallbackError::MediaTooLarge) => unreachable!("handled inside upload"),
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
@@ -507,33 +614,63 @@ pub async fn send_animation(bot: &Bot, task: &Task) -> Result<Vec<i64>, SendErro
|
||||
.await
|
||||
{
|
||||
Ok(message) => Ok(vec![message.id.0 as i64]),
|
||||
Err(RequestError::Api(api)) if is_media_fetch_failure(&api) => {
|
||||
Err(RequestError::Api(api))
|
||||
if is_media_fetch_failure(&api) || is_size_error(&api) =>
|
||||
{
|
||||
log::info!(
|
||||
"Telegram could not fetch animation URL, downloading and reuploading: {}",
|
||||
media_url
|
||||
);
|
||||
let file = match download_to_temp(animation).await {
|
||||
Ok(file) => file,
|
||||
match download_to_temp(animation).await {
|
||||
Ok(file) => {
|
||||
let path = file.path().to_path_buf();
|
||||
match send_animation_inner(
|
||||
bot,
|
||||
chat_id,
|
||||
reply_to,
|
||||
caption,
|
||||
has_spoiler,
|
||||
InputFile::file(path),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(message) => Ok(vec![message.id.0 as i64]),
|
||||
Err(e) => Err(classify_to_send_error(&e, task.clone())),
|
||||
}
|
||||
}
|
||||
// Over the upload cap: fall back to the smaller URL.
|
||||
Err(FallbackError::MediaTooLarge) => match animation.fallback_url() {
|
||||
Some(url) => match input_file_for(url) {
|
||||
Ok(file) => {
|
||||
match send_animation_inner(
|
||||
bot,
|
||||
chat_id,
|
||||
reply_to,
|
||||
caption,
|
||||
has_spoiler,
|
||||
file,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(message) => Ok(vec![message.id.0 as i64]),
|
||||
Err(e) => Err(classify_to_send_error(&e, task.clone())),
|
||||
}
|
||||
}
|
||||
Err(message) => {
|
||||
Err(SendError::Permanent { message, task: task.clone() })
|
||||
}
|
||||
},
|
||||
None => Err(SendError::Permanent {
|
||||
message: "media too large".into(),
|
||||
task: task.clone(),
|
||||
}),
|
||||
},
|
||||
Err(FallbackError::Retryable { delay_seconds }) => {
|
||||
return Err(SendError::Retryable { delay_seconds, task: task.clone() });
|
||||
Err(SendError::Retryable { delay_seconds, task: task.clone() })
|
||||
}
|
||||
Err(FallbackError::Permanent { message }) => {
|
||||
return Err(SendError::Permanent { message, task: task.clone() });
|
||||
Err(SendError::Permanent { message, task: task.clone() })
|
||||
}
|
||||
};
|
||||
let path = file.path().to_path_buf();
|
||||
match send_animation_inner(
|
||||
bot,
|
||||
chat_id,
|
||||
reply_to,
|
||||
caption,
|
||||
has_spoiler,
|
||||
InputFile::file(path),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(message) => Ok(vec![message.id.0 as i64]),
|
||||
Err(e) => Err(classify_to_send_error(&e, task.clone())),
|
||||
}
|
||||
}
|
||||
Err(e) => Err(classify_to_send_error(&e, task.clone())),
|
||||
@@ -819,6 +956,36 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_size_error_matches_known_errors() {
|
||||
// 413 upload cap.
|
||||
let e = ApiError::RequestEntityTooLarge;
|
||||
assert!(is_size_error(&e), "{e:?}");
|
||||
// Unknown descriptions with size wording.
|
||||
for description in [
|
||||
"Bad Request: file is too large",
|
||||
"Bad Request: media is too big",
|
||||
"Bad Request: url file size is too big",
|
||||
] {
|
||||
let api = ApiError::Unknown(description.to_string());
|
||||
assert!(is_size_error(&api), "{description}");
|
||||
}
|
||||
// Unrelated errors must not match.
|
||||
for description in ["Bad Request: WEBPAGE_MEDIA_EMPTY", "Bad Request: message is not modified"] {
|
||||
let api = ApiError::Unknown(description.to_string());
|
||||
assert!(!is_size_error(&api), "{description}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn media_item_payload_fallback_url_serde_default() {
|
||||
// Old queued payloads without the field deserialize with None.
|
||||
let json = serde_json::json!({"kind": "photo", "media": "https://a/b.jpg", "has_spoiler": false});
|
||||
let photo: MediaItemPayload = serde_json::from_value(json).unwrap();
|
||||
assert!(matches!(photo, MediaItemPayload::Photo { fallback_url: None, .. }));
|
||||
assert_eq!(photo.fallback_url(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classification_mapping() {
|
||||
use teloxide::types::Seconds;
|
||||
@@ -858,11 +1025,13 @@ mod tests {
|
||||
vec![MediaItemPayload::Photo {
|
||||
media: "https://a/b.jpg".into(),
|
||||
has_spoiler: true,
|
||||
fallback_url: Some("https://a/b_small.jpg".into()),
|
||||
}],
|
||||
vec![MediaItemPayload::Video {
|
||||
media: "https://a/v.mp4".into(),
|
||||
has_spoiler: false,
|
||||
thumbnail: Some("https://a/t.jpg".into()),
|
||||
fallback_url: None,
|
||||
}],
|
||||
],
|
||||
batch_index: 1,
|
||||
@@ -900,6 +1069,7 @@ mod tests {
|
||||
let photo = MediaItemPayload::Photo {
|
||||
media: "https://a/b.jpg".into(),
|
||||
has_spoiler: false,
|
||||
fallback_url: None,
|
||||
};
|
||||
let json = serde_json::to_value(&photo).unwrap();
|
||||
assert_eq!(json["kind"], "photo");
|
||||
|
||||
+52
-14
@@ -1,30 +1,68 @@
|
||||
services:
|
||||
nginx-proxy:
|
||||
image: nginxproxy/nginx-proxy:1.11.6-alpine
|
||||
restart: always
|
||||
ports:
|
||||
- '80:80'
|
||||
- '443:443'
|
||||
environment:
|
||||
# Bare-IP access only.
|
||||
# DEFAULT_HOST: 'bot.example.com'
|
||||
volumes:
|
||||
- /var/run/docker.sock:/tmp/docker.sock:ro
|
||||
- ./nginx-certs:/etc/nginx/certs:ro
|
||||
- ./nginx-vhost.d:/etc/nginx/vhost.d:ro
|
||||
- ./nginx-html:/usr/share/nginx/html:ro
|
||||
networks: [proxy]
|
||||
labels:
|
||||
- 'com.github.jrcs.letsencrypt_nginx_proxy_companion.nginx_proxy=true'
|
||||
container_name: nginx-proxy
|
||||
|
||||
acme-companion:
|
||||
image: nginxproxy/acme-companion
|
||||
restart: always
|
||||
environment:
|
||||
DEFAULT_EMAIL: 'admin@yoursfunny.top'
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
- ./nginx-certs:/etc/nginx/certs:rw
|
||||
- ./nginx-vhost.d:/etc/nginx/vhost.d:rw
|
||||
- ./nginx-html:/usr/share/nginx/html:rw
|
||||
- ./nginx-acme:/etc/acme.sh
|
||||
networks: [proxy]
|
||||
container_name: acme-companion
|
||||
depends_on:
|
||||
- nginx-proxy
|
||||
|
||||
tgxmb:
|
||||
image: yoursfunny/telegram-twitter-media-bot:latest
|
||||
restart: always
|
||||
# ports:
|
||||
# - "8443:8443"
|
||||
environment:
|
||||
# docker-entrypoint.sh drops privileges to this uid.
|
||||
LOCAL_USER_ID: '1000'
|
||||
# Bot token (BotFather). Required.
|
||||
TELOXIDE_TOKEN: ''
|
||||
# Comma-separated admin chat ids; receives startup/shutdown notices.
|
||||
BOT_ADMIN: ''
|
||||
# Required for pixiv support; pixiv is disabled when unset.
|
||||
PIXIV_REFRESH_TOKEN: ''
|
||||
# Edit-before-forward records expire after this many seconds (default 86400 = 24h).
|
||||
# Optional: x.com session cookie (auth_token) — fetches NSFW tweets
|
||||
# that the public syndication endpoint withholds.
|
||||
TWITTER_AUTH_TOKEN: ''
|
||||
EDIT_MESSAGE_TTL_SECONDS: '86400'
|
||||
RUST_LOG: 'info'
|
||||
# Webhook mode is off by default (polling). The listener binds inside the
|
||||
# container, so use 0.0.0.0 and publish the port if you enable it.
|
||||
WEBHOOK: 'false'
|
||||
VIRTUAL_HOST: 'bot.example.com'
|
||||
VIRTUAL_PORT: '8443'
|
||||
# LETSENCRYPT_HOST: 'bot.example.com'
|
||||
WEBHOOK: 'true'
|
||||
WEBHOOK_LISTEN: '0.0.0.0'
|
||||
WEBHOOK_PORT: '8443'
|
||||
WEBHOOK_URL: 'https://example.com'
|
||||
WEBHOOK_CERT: './cert/cert.pem'
|
||||
WEBHOOK_SECRET_TOKEN: 'secret-token'
|
||||
WEBHOOK_URL: 'https://bot.example.com/'
|
||||
# WEBHOOK_CERT: './cert/cert.pem'
|
||||
WEBHOOK_SECRET_TOKEN: ''
|
||||
volumes:
|
||||
- ./data:/app/data
|
||||
# - ./cert:/app/cert
|
||||
networks: [proxy]
|
||||
depends_on:
|
||||
- nginx-proxy
|
||||
container_name: tgxmb
|
||||
|
||||
networks:
|
||||
proxy:
|
||||
name: proxy
|
||||
|
||||
Reference in New Issue
Block a user