mirror of
https://github.com/TheFunny/TelegramTwitterMediaBot.git
synced 2026-09-23 23:32:05 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9e873131d4
|
||
|
|
5b77d14497
|
||
|
|
d4c36feb9a
|
||
|
|
5d0acdac01
|
||
|
|
d6707133cc
|
||
|
|
fb601f4d5d
|
||
|
|
d8dd4fa91e
|
||
|
|
af96caff40
|
||
|
|
52184ba6fb
|
||
|
|
0eb4e5c78d
|
||
|
|
5c51de217a
|
||
|
|
c1f5d3ca54
|
||
|
|
1bb6968108 | ||
|
|
14b444d109
|
||
|
|
de3105d4cd | ||
|
|
c46103a23f | ||
|
|
9529f64b41
|
||
|
|
d0de17329d | ||
|
|
9af37e92b4 | ||
|
|
e68a1dbd30 | ||
|
|
b9c6d16ff0
|
||
|
|
ec65c3ce74
|
||
|
|
893ab7a1e0
|
||
|
|
bd032e3d68
|
||
|
|
dbda6ec1c2
|
||
|
|
0a9ff58a69
|
||
|
|
c2d7c8406e
|
||
|
|
abdc27ed5e
|
||
|
|
3fb8421c3a
|
||
|
|
475cfd18f9
|
||
|
|
0a577600fd
|
@@ -26,6 +26,10 @@
|
|||||||
*.db
|
*.db
|
||||||
LICENSE
|
LICENSE
|
||||||
README.md
|
README.md
|
||||||
|
# Documentation and scratch files: the build only ever reads the manifests,
|
||||||
|
# `crates/` and the entrypoint script.
|
||||||
|
docs/
|
||||||
|
*.md
|
||||||
data/
|
data/
|
||||||
cert/
|
cert/
|
||||||
nginx-certs/
|
nginx-certs/
|
||||||
|
|||||||
@@ -0,0 +1,32 @@
|
|||||||
|
version: 2
|
||||||
|
|
||||||
|
# Pairs with the `actions-rust-lang/audit` gate in ci.yml: the gate reports
|
||||||
|
# advisories in Cargo.lock, this is what actually moves the dependencies.
|
||||||
|
# Patch bumps are batched into one PR; minor/major stay separate so they get
|
||||||
|
# reviewed and tested individually.
|
||||||
|
updates:
|
||||||
|
- package-ecosystem: cargo
|
||||||
|
directory: /
|
||||||
|
schedule:
|
||||||
|
interval: weekly
|
||||||
|
open-pull-requests-limit: 5
|
||||||
|
groups:
|
||||||
|
cargo-patch:
|
||||||
|
applies-to: version-updates
|
||||||
|
patterns: ['*']
|
||||||
|
update-types: ['patch']
|
||||||
|
|
||||||
|
# The workflow actions are pinned to commit SHAs; that pin is what makes
|
||||||
|
# bumping them a manual chore, so let the bot do it.
|
||||||
|
- package-ecosystem: github-actions
|
||||||
|
directory: /
|
||||||
|
schedule:
|
||||||
|
interval: weekly
|
||||||
|
open-pull-requests-limit: 5
|
||||||
|
|
||||||
|
# The Dockerfile's base images (rust:1-bookworm, debian:bookworm-slim).
|
||||||
|
- package-ecosystem: docker
|
||||||
|
directory: /
|
||||||
|
schedule:
|
||||||
|
interval: monthly
|
||||||
|
open-pull-requests-limit: 3
|
||||||
+49
-14
@@ -4,15 +4,19 @@ name: CI
|
|||||||
# job that exercises the real source sites and the token-gated pixiv tests.
|
# job that exercises the real source sites and the token-gated pixiv tests.
|
||||||
#
|
#
|
||||||
# Layering:
|
# Layering:
|
||||||
# test — fmt + clippy + the full offline unit suite + cargo-audit
|
# test — fmt + clippy + the full offline unit suite + a release-profile
|
||||||
# dependency gate. Runs on every push and PR, including forks
|
# build + cargo-audit dependency gate. Runs on every push and PR,
|
||||||
# (it needs no secrets).
|
# including forks (it needs no secrets).
|
||||||
# live — the #[ignore]d live-network tests plus the pixiv tests that are
|
# live — the #[ignore]d live-network tests plus the pixiv tests that are
|
||||||
# gated on PIXIV_REFRESH_TOKEN. Runs on schedule / manual dispatch
|
# gated on PIXIV_REFRESH_TOKEN. Runs on schedule / manual dispatch
|
||||||
# / tag pushes only, because pull requests from forks cannot read
|
# / tag pushes only, because pull requests from forks cannot read
|
||||||
# repository secrets. continue-on-error keeps a flaky external site
|
# repository secrets. continue-on-error keeps a flaky external site
|
||||||
# from blocking, while the run still records the outcome.
|
# from blocking, while the run still records the outcome.
|
||||||
#
|
#
|
||||||
|
# Every action is pinned to a commit SHA (Dependabot keeps the pins current);
|
||||||
|
# `dtolnay/rust-toolchain` deliberately stays on its channel ref, because the
|
||||||
|
# ref itself is what selects the toolchain (`@stable` = install stable).
|
||||||
|
#
|
||||||
# Test gating convention (keep in sync with AGENTS.md "Testing & QA"):
|
# Test gating convention (keep in sync with AGENTS.md "Testing & QA"):
|
||||||
# - pure unit tests: plain #[test] / #[tokio::test], always run.
|
# - pure unit tests: plain #[test] / #[tokio::test], always run.
|
||||||
# - live-network tests: #[ignore = "live network: ..."], only run here.
|
# - live-network tests: #[ignore = "live network: ..."], only run here.
|
||||||
@@ -28,44 +32,75 @@ on:
|
|||||||
- cron: '0 3 * * 1'
|
- cron: '0 3 * * 1'
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
# A newer push to the same ref supersedes the older run; without this every
|
||||||
|
# intermediate commit of a PR branch keeps a runner busy to completion.
|
||||||
|
concurrency:
|
||||||
|
group: ci-${{ github.ref }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
env:
|
||||||
|
# Panicking tests print their backtrace; free when nothing fails.
|
||||||
|
RUST_BACKTRACE: 1
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
test:
|
test:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
# Generous on purpose: the release-profile build below is cold on the very
|
||||||
|
# first run (thin LTO + codegen-units = 1 across every dependency), and a
|
||||||
|
# timeout there would kill the job *before* rust-cache saves its cache —
|
||||||
|
# leaving every later run cold again.
|
||||||
|
timeout-minutes: 45
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v7
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||||
- uses: dtolnay/rust-toolchain@stable
|
- uses: dtolnay/rust-toolchain@stable
|
||||||
with:
|
with:
|
||||||
components: clippy, rustfmt
|
components: clippy, rustfmt
|
||||||
- uses: Swatinem/rust-cache@v2
|
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
|
||||||
|
# `--locked` on every cargo invocation: the version bump edits
|
||||||
|
# Cargo.lock by hand (AGENTS.md), so a stale lock must fail here instead
|
||||||
|
# of being silently re-resolved — otherwise CI tests a different
|
||||||
|
# dependency set than the one committed, and than the one the released
|
||||||
|
# image is built from.
|
||||||
- name: Check formatting
|
- name: Check formatting
|
||||||
run: cargo fmt --check
|
run: cargo fmt --check
|
||||||
- name: Lint (deny warnings)
|
- name: Lint (deny warnings)
|
||||||
run: cargo clippy --workspace --all-targets -- -D warnings
|
run: cargo clippy --workspace --all-targets --locked -- -D warnings
|
||||||
- name: Run offline tests
|
- name: Run offline tests
|
||||||
run: cargo test --workspace
|
run: cargo test --workspace --locked
|
||||||
|
# The release profile (lto/strip/codegen-units=1, overflow checks off)
|
||||||
|
# was otherwise only exercised by the Docker build on master/tag. Same
|
||||||
|
# package the Dockerfile builds; the cache keeps it cheap after the
|
||||||
|
# first run.
|
||||||
|
- name: Build release profile
|
||||||
|
run: cargo build --release --locked -p xmedia-bot
|
||||||
# Dependency vulnerability gate: fails the build when a crate in
|
# Dependency vulnerability gate: fails the build when a crate in
|
||||||
# Cargo.lock has an unfixed security advisory. Unmaintained/unsound
|
# Cargo.lock has an unfixed security advisory. Unmaintained/unsound
|
||||||
# *warnings* (dotenv, proc-macro-error2, anyhow transitive) do not fail
|
# *warnings* (dotenv, proc-macro-error2, anyhow transitive) do not fail
|
||||||
# the build by default; the advisory DB is cached across runs.
|
# the build by default; the advisory DB is cached across runs.
|
||||||
- name: Audit dependencies
|
- name: Audit dependencies
|
||||||
uses: actions-rust-lang/audit@v1
|
uses: actions-rust-lang/audit@72c09e02f132669d52284a3323acdb503cfc1a24 # v1
|
||||||
|
|
||||||
live:
|
live:
|
||||||
needs: test
|
needs: test
|
||||||
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/v')
|
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/v')
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 30
|
||||||
continue-on-error: true
|
continue-on-error: true
|
||||||
env:
|
env:
|
||||||
PIXIV_REFRESH_TOKEN: ${{ secrets.PIXIV_REFRESH_TOKEN }}
|
PIXIV_REFRESH_TOKEN: ${{ secrets.PIXIV_REFRESH_TOKEN }}
|
||||||
TWITTER_AUTH_TOKEN: ${{ secrets.TWITTER_AUTH_TOKEN }}
|
TWITTER_AUTH_TOKEN: ${{ secrets.TWITTER_AUTH_TOKEN }}
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v7
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||||
- uses: dtolnay/rust-toolchain@stable
|
- uses: dtolnay/rust-toolchain@stable
|
||||||
- uses: Swatinem/rust-cache@v2
|
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
|
||||||
# Full suite: with the secret present, the pixiv token-gated tests run;
|
# Everything network- or secret-gated lives in x-media, and the bot
|
||||||
# without it they skip themselves. Live tests stay #[ignore]d here.
|
# crate's suite (MockSender + tempdir stores, no network) already ran in
|
||||||
|
# the `test` job — rebuilding it here bought nothing.
|
||||||
- name: Run token-gated tests
|
- name: Run token-gated tests
|
||||||
run: cargo test --workspace
|
run: cargo test -p x-media --locked
|
||||||
# The live-network tests, by the "live" name filter (all #[ignore]d).
|
# The live-network tests, by the "live" name filter (all #[ignore]d).
|
||||||
- name: Run live-network tests
|
- name: Run live-network tests
|
||||||
run: cargo test --workspace -- --ignored live
|
run: cargo test -p x-media --locked -- --ignored live
|
||||||
|
|||||||
@@ -1,28 +1,75 @@
|
|||||||
name: Build Docker Image
|
name: Build Docker Image
|
||||||
|
|
||||||
|
# Release builds (master / v* tags) plus a build-only check on pull requests
|
||||||
|
# that touch anything the image depends on — the Dockerfile's stub-source
|
||||||
|
# machinery, the ffmpeg download and the entrypoint are exactly the parts that
|
||||||
|
# would otherwise break only at release time.
|
||||||
|
#
|
||||||
|
# Actions are pinned to commit SHAs (Dependabot keeps the pins current).
|
||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
tags:
|
tags:
|
||||||
- v*
|
- v*
|
||||||
branches:
|
branches:
|
||||||
- master
|
- master
|
||||||
|
pull_request:
|
||||||
|
paths:
|
||||||
|
- Dockerfile
|
||||||
|
- docker-entrypoint.sh
|
||||||
|
- .dockerignore
|
||||||
|
- Cargo.toml
|
||||||
|
- Cargo.lock
|
||||||
|
- .github/workflows/docker.yml
|
||||||
|
- 'crates/**/Cargo.toml'
|
||||||
|
|
||||||
env:
|
env:
|
||||||
APP_NAME: telegram-twitter-media-bot
|
APP_NAME: telegram-twitter-media-bot
|
||||||
DOCKERHUB_REPO: yoursfunny/telegram-twitter-media-bot
|
DOCKERHUB_REPO: yoursfunny/telegram-twitter-media-bot
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
# Serialize runs per ref. Never cancel in progress: a killed run would drop a
|
||||||
|
# half-finished image push.
|
||||||
|
concurrency:
|
||||||
|
group: docker-${{ github.ref }}
|
||||||
|
cancel-in-progress: false
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
# A tag push and a branch push to the same commit fire two workflow runs;
|
# A tag push and a branch push to the same commit fire two workflow runs;
|
||||||
# build only once. Tag runs always build; master runs build only when the
|
# build only once. Tag runs always build; master runs build only when the
|
||||||
# pushed commit is not already tagged (the tag run covers it).
|
# pushed commit is not already tagged (the tag run covers it).
|
||||||
should-build:
|
should-build:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 10
|
||||||
outputs:
|
outputs:
|
||||||
build: ${{ steps.check.outputs.build }}
|
build: ${{ steps.check.outputs.build }}
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v7
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
|
# A release tag is the version claim: the manifests are bumped by hand,
|
||||||
|
# so `v1.5.1` with `Cargo.toml` still at 1.5.0 would publish an image
|
||||||
|
# whose tag lies about what is inside it (the binary carries no version).
|
||||||
|
- name: Verify the tag matches both crate versions
|
||||||
|
if: startsWith(github.ref, 'refs/tags/v')
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
tag="${GITHUB_REF_NAME#v}"
|
||||||
|
status=0
|
||||||
|
for manifest in crates/x-media/Cargo.toml crates/xmedia-bot/Cargo.toml; do
|
||||||
|
# tr -d '\r': a CRLF checkout (core.autocrlf on Windows) would
|
||||||
|
# otherwise yield "1.5.0\r" and false-fail every tag.
|
||||||
|
version="$(sed -n 's/^version = "\(.*\)"/\1/p' "$manifest" | head -1 | tr -d '\r')"
|
||||||
|
if [ "$version" != "$tag" ]; then
|
||||||
|
echo "::error file=$manifest::$manifest is at $version but the tag is v$tag"
|
||||||
|
status=1
|
||||||
|
else
|
||||||
|
echo "$manifest: $version matches v$tag"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
exit "$status"
|
||||||
- id: check
|
- id: check
|
||||||
shell: bash
|
shell: bash
|
||||||
run: |
|
run: |
|
||||||
@@ -33,14 +80,20 @@ jobs:
|
|||||||
echo "build=true" >> "$GITHUB_OUTPUT"
|
echo "build=true" >> "$GITHUB_OUTPUT"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# No `actions/checkout` here on purpose: `docker/build-push-action` defaults
|
||||||
|
# to the Git context (`https://github.com/<owner>/<repo>.git#<ref>`), so
|
||||||
|
# BuildKit clones the repo itself and authenticates with the automatic
|
||||||
|
# github.token. Adding `context: .` below without a checkout step would hand
|
||||||
|
# BuildKit an empty workspace.
|
||||||
docker:
|
docker:
|
||||||
needs: should-build
|
needs: should-build
|
||||||
if: needs.should-build.outputs.build == 'true'
|
if: needs.should-build.outputs.build == 'true'
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 60
|
||||||
steps:
|
steps:
|
||||||
- name: Docker meta
|
- name: Docker meta
|
||||||
id: meta
|
id: meta
|
||||||
uses: docker/metadata-action@v6
|
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6
|
||||||
with:
|
with:
|
||||||
images: ${{ env.DOCKERHUB_REPO }}
|
images: ${{ env.DOCKERHUB_REPO }}
|
||||||
tags: |
|
tags: |
|
||||||
@@ -49,15 +102,15 @@ jobs:
|
|||||||
type=semver,pattern={{major}}.{{minor}}
|
type=semver,pattern={{major}}.{{minor}}
|
||||||
type=semver,pattern={{major}}
|
type=semver,pattern={{major}}
|
||||||
type=sha
|
type=sha
|
||||||
-
|
|
||||||
name: Set up QEMU
|
|
||||||
uses: docker/setup-qemu-action@v4
|
|
||||||
-
|
-
|
||||||
name: Set up Docker Buildx
|
name: Set up Docker Buildx
|
||||||
uses: docker/setup-buildx-action@v4
|
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
|
||||||
|
# Pull requests build the image to prove the Dockerfile still works, but
|
||||||
|
# must not read registry credentials (fork PRs have none).
|
||||||
-
|
-
|
||||||
name: Login to Docker Hub
|
name: Login to Docker Hub
|
||||||
uses: docker/login-action@v4
|
if: github.event_name != 'pull_request'
|
||||||
|
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
|
||||||
with:
|
with:
|
||||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||||
@@ -66,15 +119,26 @@ jobs:
|
|||||||
# stage's layers so the cargo-deps and ffmpeg layers are restored
|
# stage's layers so the cargo-deps and ffmpeg layers are restored
|
||||||
# instead of re-downloaded/recompiled. The scope must be pinned to a
|
# instead of re-downloaded/recompiled. The scope must be pinned to a
|
||||||
# fixed string: the gha backend defaults to the current git ref, which
|
# fixed string: the gha backend defaults to the current git ref, which
|
||||||
# would give every new tag a cold cache on release builds.
|
# would give every new tag a cold cache on release builds. PR runs only
|
||||||
|
# read it (cache-to is empty) so they cannot evict the release cache.
|
||||||
|
#
|
||||||
|
# FFMPEG_URL/FFMPEG_SHA256 come from repository variables when set, so a
|
||||||
|
# release can pin an exact ffmpeg build (the Dockerfile default follows
|
||||||
|
# the project's `/redirect/latest/` URL, which has no sha256 sidecar).
|
||||||
|
#
|
||||||
|
# Single-arch (amd64) on purpose: adding arm64 means re-adding
|
||||||
|
# `docker/setup-qemu-action`, `platforms: linux/amd64,linux/arm64`, and
|
||||||
|
# parameterizing FFMPEG_URL by $TARGETARCH in the Dockerfile.
|
||||||
-
|
-
|
||||||
name: Build and push
|
name: Build and push
|
||||||
uses: docker/build-push-action@v7
|
uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7
|
||||||
with:
|
with:
|
||||||
push: true
|
push: ${{ github.event_name != 'pull_request' }}
|
||||||
build-args: |
|
build-args: |
|
||||||
APP_NAME=${{ env.APP_NAME }}
|
APP_NAME=${{ env.APP_NAME }}
|
||||||
|
FFMPEG_URL=${{ vars.FFMPEG_URL || 'https://ffmpeg.martin-riedl.de/redirect/latest/linux/amd64/release/ffmpeg.zip' }}
|
||||||
|
FFMPEG_SHA256=${{ vars.FFMPEG_SHA256 }}
|
||||||
tags: ${{ steps.meta.outputs.tags }}
|
tags: ${{ steps.meta.outputs.tags }}
|
||||||
labels: ${{ steps.meta.outputs.labels }}
|
labels: ${{ steps.meta.outputs.labels }}
|
||||||
cache-from: type=gha,scope=tgxmb-build
|
cache-from: type=gha,scope=tgxmb-build
|
||||||
cache-to: type=gha,mode=max,scope=tgxmb-build
|
cache-to: ${{ github.event_name != 'pull_request' && 'type=gha,mode=max,scope=tgxmb-build' || '' }}
|
||||||
|
|||||||
@@ -2,9 +2,9 @@
|
|||||||
|
|
||||||
## Project Overview
|
## Project Overview
|
||||||
|
|
||||||
Telegram bot (teloxide) that turns post links from X/Twitter, Pixiv, Bluesky, and Misskey (misskey.io) into media messages (images, video, GIF) with the post's title, author, and tags. It supports batch media splitting, retry with persistence, inline queries, forward-channel rebinding with caption templates, and Pixiv ugoira→MP4 transcoding. README is in Chinese; user-facing bot strings are in English. The project is a Rust port of a Python predecessor (see `queue.rs` comments referencing `utils/task_queue.py`).
|
Telegram bot (teloxide) that turns post links from X/Twitter, Pixiv, Bluesky, Misskey (misskey.io), and Bilibili dynamics into media messages (images, video, GIF) with the post's title, author, and tags. It supports batch media splitting, retry with persistence, inline queries, forward-channel rebinding with caption templates, and Pixiv ugoira→MP4 transcoding. README is in Chinese; user-facing bot strings are in English. The project is a Rust port of a Python predecessor (see `queue.rs` comments referencing `utils/task_queue.py`).
|
||||||
|
|
||||||
Two-crate Cargo workspace (both v1.5.0, edition 2024, resolver 3):
|
Two-crate Cargo workspace (both v1.8.0, edition 2024, resolver 3):
|
||||||
|
|
||||||
- **`crates/x-media`** — library that fetches and normalizes media from the four sites. Pure, no Telegram knowledge.
|
- **`crates/x-media`** — library that fetches and normalizes media from the four sites. Pure, no Telegram knowledge.
|
||||||
- **`crates/xmedia-bot`** — the bot binary: teloxide dispatcher, SQLite-backed chat state, persistent task queue.
|
- **`crates/xmedia-bot`** — the bot binary: teloxide dispatcher, SQLite-backed chat state, persistent task queue.
|
||||||
@@ -18,27 +18,36 @@ Telegram update → Dispatcher (polling or axum webhook) → dptree branches
|
|||||||
└─ callback_query → "forward" (copy to channel) / "template|<name>" (apply caption template)
|
└─ callback_query → "forward" (copy to channel) / "template|<name>" (apply caption template)
|
||||||
```
|
```
|
||||||
|
|
||||||
Message flow: `message_handler` extracts URLs (from `url`/`text_link` entities, text + caption, deduped) → `x_media::site::fetch(url)` → `Fetched` → builds a `Task` → `send::send_media_sequence` (media groups ≤ 9, caption on first item) or `send::send_animation`. On Telegram URL-fetch failure or size error (`send_batch_via_upload`): download via `x_media::site::download_media` to a temp file (≤ 10 MiB), sniff magic bytes (`sniff_ext`), upload via multipart; oversized items fall back to `fallback_url`. On failure: `enqueue_retry` persists resume-state `Task` into the SQLite queue → workers lease (120 s lock TTL) → retry with exponential backoff (≤ 30 s, `MAX_RETRIES = 2`) → dead-letter → `notify_failure`. Success → `post_send_actions`: edit-before-forward prompt with inline buttons, or `copy_messages` to the bound forward channel.
|
Message flow: `message_handler` extracts URLs (from `url`/`text_link` entities, text + caption, deduped) → `x_media::site::fetch(url)` → `Fetched` → builds a `Task` → `send::send_media_sequence` (media groups ≤ 10, caption on first item) or `send::send_animation`. On Telegram URL-fetch failure or size error (`send_batch_via_upload`): download via `x_media::site::download_media` to a temp file (≤ 10 MiB), sniff magic bytes (`sniff_ext`), upload via multipart; oversized items fall back to `fallback_url`. On failure: `enqueue_retry` persists resume-state `Task` into the SQLite queue → workers lease (120 s lock TTL) → retry with exponential backoff (≤ 30 s, `MAX_RETRIES = 2`) → dead-letter → `notify_failure`. Success → `post_send_actions`: edit-before-forward prompt with inline buttons, or `copy_messages` to the bound forward channel.
|
||||||
|
|
||||||
Debug command: `/test <url>` runs the same `x_media::site::fetch` and replies with `test_parse_report` (`handlers/commands.rs`) — site id, normalized cache key, source URL, title/author/tags, sensitive flag, caption and the media list — nothing is sent, cached or forwarded; the report is capped at 4000 chars and sent with HTML parse mode: raw fields are escaped, and the caption is wrapped in a `<blockquote>` so it renders exactly like the sent media caption (escaped text and links included). It uses a custom `parse_test_arg` parser (whole remainder, trimmed) because teloxide's built-in `split` parser takes exactly one space-separated token.
|
Debug command: `/debug <url>` runs the same `x_media::site::fetch` and replies with `debug_report` (`handlers/commands.rs`) — site id, normalized cache key, source URL, title/author/tags, sensitive flag, caption and the media list — nothing is sent, cached or forwarded; the report is capped at 4000 chars and sent with HTML parse mode: raw fields are escaped, and the caption is wrapped in a `<blockquote>` so it renders exactly like the sent media caption (escaped text and links included). The caption it shows is `preview_caption`'s: the chat's per-site format override plus the long-post quoting, i.e. exactly what the send paths produce — showing the raw built-in caption made `/set_format` look like a no-op, and the `/set_format` success reply points users at `/debug` to preview.
|
||||||
|
|
||||||
The `x-media` library: `site::fetch(url)` dispatches through the `SITES` registry (per-site `impl Site`, in order twitter → bsky → misskey → pixiv) and returns `Ok(None)` for unmatched URLs. `Fetched { source_url, caption, title, media: Vec<Media>, sensitive, site_id, … }`; `caption_with(format)` substitutes `{url} {author} {author_url} {title} {tags}`.
|
User-facing failure text is a function of the error class, never one generic sentence: `urls::fetch_error_message` maps `FetchError::NotFound` (post gone), `Sensitive` (withheld, needs `TWITTER_AUTH_TOKEN`), `Blocked` (source risk control), `Disabled { site }` (a registered site switched off — pixiv without a token, the one case `fetch` answers `Err` instead of `Ok(None)`) and `Transient`/`Http` (source down) apart. The same distinction drives the group hint: a supported link posted in a group (not a channel) gets one `GROUP_LINK_HINT` reply, because the link pipeline is private-chat only.
|
||||||
|
|
||||||
|
The `/test <url>` command runs the ordinary link pipeline (`urls::url_media`) with `PostSend::Suppressed`: the media is sent and cached like any other link, but the chat's `forward_channel_id`/`edit_before_forward` are ignored, so a test never forwards to the channel and never opens the edit prompt (retries and dead-letter notifications behave as usual). `/test`, `/debug`, `/set_format` and `/clear_cache` use the custom `parse_arg_remainder` parser (whole remainder, trimmed) because teloxide's built-in `split` parser takes exactly one space-separated token per field: `/set_format <site> <format>` never parsed with it (and `/clear_cache` without an argument did not either), and a command that fails to parse falls through to the URL flow in silence. `commands::tests::every_documented_invocation_parses` pins every documented form against exactly that.
|
||||||
|
|
||||||
|
The inline path (`handlers/inline.rs`) hands media URLs straight to Telegram, which fetches them itself and cannot send site-specific headers — so `x_media::site::needs_media_headers(url)` (true exactly where a site's `media_headers` is non-empty, i.e. pixiv's pximg.net) marks the media that must be skipped instead of shipped broken; locally produced media (ugoira MP4, bsky remux) fails `Url::parse` and is skipped the same way. Inline results are therefore URL-only by construction.
|
||||||
|
|
||||||
|
`url_media` is a thin wrapper over `url_media_inner`: `run_with_chat_action` sends the chat action, then re-sends it every `ACTION_REFRESH` (4 s) while the pipeline future is pending, because Telegram drops an action after ~5 s and a fetch (ugoira encode, HLS remux) plus an upload routinely outlasts that. The pipeline flips the shared `ActionHint` from `Typing` to `UploadPhoto`/`UploadVideo` once the media kinds are known. The `select!` is `biased` on the pipeline branch so a finished pipeline never emits a stray action.
|
||||||
|
|
||||||
|
The `x-media` library: `site::fetch(url)` dispatches through the `SITES` registry (per-site `impl Site`, in order twitter → bsky → misskey → pixiv → bilibili) and returns `Ok(None)` for unmatched URLs (`Err(FetchError::Disabled { site })` when the URL matches a registered site whose `enabled()` is false — see `disabled_site`). `Fetched { source_url, caption, title, content, media: Vec<Media>, sensitive, site_id, … }` (title and content are split per platform: a pixiv artwork's title and description, a bilibili headline and body, and text-only posts whose text is all `content`); `caption_with(format)` substitutes `{url} {author} {author_url} {title} {content} {tags}`.
|
||||||
|
|
||||||
## Key Directories
|
## Key Directories
|
||||||
|
|
||||||
| Path | Purpose |
|
| Path | Purpose |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `crates/x-media/src/` | Fetch library. `site/mod.rs` = dispatcher + `Fetched`/`FetchError`/`download_media`/`media_size`; `media.rs` = `Media` enum; `examples/fetch.rs` = end-to-end usage sample |
|
| `crates/x-media/src/` | Fetch library. `site/mod.rs` = dispatcher + `Fetched`/`FetchError`/`download_media`/`media_size`; `media.rs` = `Media` enum; `examples/fetch.rs` = end-to-end usage sample |
|
||||||
| `crates/x-media/src/site/<twitter\|pixiv\|bsky\|misskey>/` | One directory per site: `mod.rs` (re-exports), `interface.rs` (PATTERN, `enabled()`, `fetch_from_url()`, `cache_key`/`is_retryable`/`media_headers`, unit struct `<Name>Site` implementing `site::Site`, `From<SiteStruct> for Fetched`), `model.rs` (serde DTOs). Pixiv adds `api.rs` (auth + transport); twitter adds `auth.rs` (logged-in GraphQL `TweetDetail` fallback for NSFW tweets, gated on `TWITTER_AUTH_TOKEN`). Misskey targets misskey.io only (`POST /api/notes/show`, 400+`NO_SUCH_NOTE` → NotFound). Twitter's `from_syndication_json` HTML-decodes the API text — syndication and GraphQL `full_text` both arrive pre-escaped (`>` `<` `&` `'`) — so the stored text is raw and the caption escapes exactly once |
|
| `crates/x-media/src/site/<twitter\|pixiv\|bsky\|misskey\|bilibili>/` | One directory per site: `mod.rs` (re-exports), `interface.rs` (PATTERN, `enabled()`, `fetch_from_url()`, `cache_key`/`is_retryable`/`media_headers`, unit struct `<Name>Site` implementing `site::Site`, `From<SiteStruct> for Fetched`), `model.rs` (serde DTOs). Pixiv adds `api.rs` (auth + transport); twitter adds `auth.rs` (logged-in GraphQL `TweetDetail` fallback for NSFW tweets, gated on `TWITTER_AUTH_TOKEN`; without the token a withheld tweet stays `FetchError::Sensitive` and the bot reports it as age-restricted instead of "no media"). Misskey targets misskey.io only (`POST /api/notes/show`, 400+`NO_SUCH_NOTE` → NotFound). Bilibili fetches dynamics (images/animated images only — an attached video degrades to its cover, and its title stands in for the post text, which AV dynamics do not have) from `/x/polymer/web-dynamic/v1/detail` sent with `features=itemOpusStyle` (without that flag the legacy serialization drops an image/text post's body and headline entirely — `desc` comes back `null`; the adapter still parses the legacy `major.draw`/`desc`/`archive` shapes as a fallback). No WBI signature is involved; device cookies `buvid3`/`buvid4` are fetched automatically from `/x/frontend/finger/spi` because bilibili's `-352` risk control starts rejecting plain requests, `BILIBILI_COOKIE` is the escalation when an IP stays blocked; `b23.tv` short links are deliberately unmatched. Twitter's `from_syndication_json` HTML-decodes the API text — syndication and GraphQL `full_text` both arrive pre-escaped (`>` `<` `&` `'`) — so the stored text is raw and the caption escap…
|
||||||
| `crates/xmedia-bot/src/main.rs` | Entry point: env/log init, command registration (`register_commands`), shared `send::BOT` force-init, queue worker start, site login validation (`site::validate_all`), 300 s edit-expiry sweep, dptree handler tree, webhook vs polling dispatch |
|
| `crates/xmedia-bot/src/main.rs` | Entry point: env/log init, command registration (`register_commands` — `setMyCommands` plus the profile description texts), shared `send::BOT` force-init, queue worker start, site login validation (`site::validate_all`), 300 s edit-expiry sweep (expired prompts are rewritten in place to `EDIT_PROMPT_EXPIRED_TEXT` with an empty keyboard — an edit, never a new message, so a background timer cannot wake a chat), dptree handler tree, webhook vs polling dispatch |
|
||||||
| `crates/xmedia-bot/src/config.rs` | Manual env parsing into `Config` |
|
| `crates/xmedia-bot/src/config.rs` | Manual env parsing into `Config` |
|
||||||
| `crates/xmedia-bot/src/db.rs` | `DbPool`: per-store SQLite connection pool (`POOL_SIZE = 4`, WAL, busy_timeout) over `$DATA_DIR/task_queue.db` (default `data/`); `open_store` creates file + schema, `with_conn` runs all rusqlite I/O in `spawn_blocking` |
|
| `crates/xmedia-bot/src/db.rs` | `DbPool`: one shared SQLite connection pool (`POOL_SIZE = 4`, WAL, busy_timeout) for all three tables over `$DATA_DIR/task_queue.db` (default `data/`) — the three stores share it; `open_store` creates file + schema, `with_conn` runs all rusqlite I/O in `spawn_blocking` |
|
||||||
| `crates/xmedia-bot/src/handlers/` | Handler modules: `mod.rs` (message entry point, `reply`, `log_key`), `commands.rs` (teloxide `BotCommands` enum + command executor, incl. the `/test <url>` parse-only debug command), `urls.rs` (URL extraction + bounded job channel (256) drained by `URL_WORKERS = 8` workers (`start_url_workers`) — backpressure instead of unbounded spawns; teloxide's per-chat workers are sequential — batch-forwards need concurrency), `inline.rs`/`callback.rs` (inline queries / edit-before-forward buttons), `statics.rs` (global statics) |
|
| `crates/xmedia-bot/src/handlers/` | Handler modules: `mod.rs` (message entry point, `reply`, `log_key`, the group-only `GROUP_LINK_HINT` for a supported link posted outside a private chat), `commands.rs` (teloxide `BotCommands` enum + command executor, incl. `/test <url>` (send-only) / `/debug <url>` (parse-only) and the admin-only `/bot_dict` state dump; `/set_format` rejects unknown `{…}` placeholders and resets with `-`), `urls.rs` (URL extraction + bounded job channel (256) drained by `URL_WORKERS = 8` workers (`start_url_workers`) — backpressure instead of unbounded spawns; teloxide's per-chat workers are sequential — batch-forwards need concurrency), `inline.rs`/`callback.rs` (inline queries / edit-before-forward buttons, incl. `skip`), `statics.rs` (global statics) |
|
||||||
| `crates/xmedia-bot/src/state.rs` | `ChatStore`: parking_lot `Mutex<HashMap>` cache + SQLite write-through (`chat_state` table) |
|
| `crates/xmedia-bot/src/state.rs` | `ChatStore`: parking_lot `Mutex<HashMap>` cache + SQLite write-through (`chat_state` table) |
|
||||||
| `crates/xmedia-bot/src/link_cache.rs` | `LinkCache`: SQLite-backed cache (`link_cache` table) of successfully sent posts — raw caption fields + Telegram `file_id`s; repeat links re-send locally (no fetch/upload), TTL + prune, invalidated on permanent send failure |
|
| `crates/xmedia-bot/src/link_cache.rs` | `LinkCache`: SQLite-backed cache (`link_cache` table) of successfully sent posts — raw caption fields + Telegram `file_id`s; repeat links re-send locally (no fetch/upload), TTL + prune, invalidated on permanent send failure |
|
||||||
| `crates/xmedia-bot/src/queue.rs` | `PersistentTaskQueue`: SQLite-backed queue (`tasks` table), `QUEUE_WORKERS = 4` concurrent workers (lease via `BEGIN IMMEDIATE` + `locked_until` TTL), retry→dead-letter, `Notify::notify_waiters` wakeup, `busy_timeout` on all connections |
|
| `crates/xmedia-bot/src/queue.rs` | `PersistentTaskQueue`: SQLite-backed queue (`tasks` table), `QUEUE_WORKERS = 4` concurrent workers (lease via `BEGIN IMMEDIATE` + `locked_until` TTL), retry→dead-letter, `notify_one` worker wakeup plus a separate `Notify` for the 30 s lease-expiry sweep (a shared one let the sweep steal the workers' wakeup permit), `busy_timeout` on all connections |
|
||||||
| `crates/xmedia-bot/src/send.rs` | Media senders, upload fallback, error classification, queue task handlers |
|
| `crates/xmedia-bot/src/ctx.rs` | `AppContext`: the injected collaborators (`sender` + `ChatStore`/`PersistentTaskQueue`/`LinkCache`/`Config`), `from_statics` for production and the `CONTEXT` static the worker closures hold. `test_support::TestStores` backs handler tests with a tempdir store set |
|
||||||
| `crates/xmedia-bot/src/media_sender.rs` | `MediaSender` trait: the send surface (`send_media_group`/`send_animation`/`copy_messages`/`send_message`/`send_chat_action`) implemented by teloxide `Bot` (per-chat rate-limited) and by a scripted `MockSender` in tests |
|
| `crates/xmedia-bot/src/send/` | `send/mod.rs`: `Task`/`MediaItemPayload` payloads, `SendError`/`Classification`, `send_media_sequence`/`send_animation`/`forward_messages`; `send/input_media.rs`: payload → `InputFile`/`InputMedia` + `build_media_group` (caption on the first item only); `send/upload.rs`: the download-and-reupload fallback (`prepare_upload_item`/`send_batch_via_upload`, photo downscale handoff); `send/post_send.rs`: link-cache write, `KEEP_ALIVE` registry, `settle_task`, `post_send_actions`, `handle_task`/`dead_letter_notify` |
|
||||||
|
| `crates/xmedia-bot/src/media_sender.rs` | `MediaSender` trait: the user-flow surface (`send_media_group`/`send_animation`/`copy_messages`/`send_message`/`answer_callback_query`/`edit_message_caption`/`delete_message`/`send_chat_action`) implemented by teloxide `Bot` (per-chat rate-limited) and by a recording `MockSender` in tests. Admin/setup APIs (`get_chat`, `set_my_commands`, …) stay on the concrete `Bot` |
|
||||||
| `crates/xmedia-bot/src/rate_limit.rs` | Per-chat token bucket (`CAPACITY = 20`, ~20 msg/min refill) paced before sends reach the API so batch forwards don't trip flood control |
|
| `crates/xmedia-bot/src/rate_limit.rs` | Per-chat token bucket (`CAPACITY = 20`, ~20 msg/min refill) paced before sends reach the API so batch forwards don't trip flood control |
|
||||||
|
|
||||||
## Development Commands
|
## Development Commands
|
||||||
@@ -57,14 +66,14 @@ Docker: `docker build -t tgxmb .` then `docker run --rm -d --name tgxmb --env-fi
|
|||||||
|
|
||||||
## Code Conventions & Common Patterns
|
## Code Conventions & Common Patterns
|
||||||
|
|
||||||
- **Errors via `thiserror` derive** (no anyhow): the public, stringified errors — `FetchError` (`Http`/`Json`/`Pixiv`/`Site`/`NotFound`/`Blocked`) and `PixivError` — derive `thiserror::Error` with `#[from]` conversions; `Display`/`source()` come from the derive. The internal control-flow enums — `QueueError` (`Retryable { delay_seconds, payload }` / `Permanent`), `SendError` (Retryable/Permanent), `Classification`, `FallbackError` — carry no `Display` and are handled by direct variant matching. New errors should follow the same split: stringified/public errors derive `thiserror`, internal flow enums stay plain.
|
- **Errors via `thiserror` derive** (no anyhow): the public, stringified errors — `FetchError` (`Http`/`Json`/`Pixiv`/`Site`/`NotFound`/`Blocked`/`Disabled`/`Sensitive`/`TooLarge`/`Transient`/`Io`) and `PixivError` — derive `thiserror::Error` with `#[from]` conversions; `Display`/`source()` come from the derive. The internal control-flow enums — `QueueError` (`Retryable { delay_seconds, payload }` / `Permanent`), `SendError` (Retryable/Permanent), `Classification`, `FallbackError` — carry no `Display` and are handled by direct variant matching. New errors should follow the same split: stringified/public errors derive `thiserror`, internal flow enums stay plain.
|
||||||
- **Global state via `std::sync::LazyLock` statics**, not DI: `CONFIG`, `CHAT_STORE`, `TASK_QUEUE` in `handlers/statics.rs`; shared reqwest `CLIENT` in `x-media/src/site/mod.rs`. `Bot` is passed/cloned into handlers; queue workers share the process-wide `send::BOT` (`LazyLock<Bot>`, force-initialized in `main` so a missing token fails at startup).
|
- **Global state via `std::sync::LazyLock` statics**, not DI: `CONFIG`, `CHAT_STORE`, `TASK_QUEUE` in `handlers/statics.rs`; shared reqwest `CLIENT` in `x-media/src/site/mod.rs`. `Bot` is passed/cloned into handlers; queue workers share the process-wide `send::BOT` (`LazyLock<Bot>`, force-initialized in `main` so a missing token fails at startup).
|
||||||
- **Async**: tokio multi-thread runtime (`#[tokio::main]` default). All rusqlite I/O inside `tokio::task::spawn_blocking`. Long loops use `tokio::select!` with `tokio::sync::{watch, Notify}` stop/wake channels. No streams.
|
- **Async**: tokio multi-thread runtime (`#[tokio::main]` default). All rusqlite I/O inside `tokio::task::spawn_blocking`. Long loops use `tokio::select!` with `tokio::sync::{watch, Notify}` stop/wake channels. No streams.
|
||||||
- **Blocking sync primitives**: `parking_lot::Mutex` for hot caches, `tokio::sync::Mutex` for async-shared state (pixiv token cache), `AtomicBool` for feature gates.
|
- **Blocking sync primitives**: `parking_lot::Mutex` for hot caches, `tokio::sync::Mutex` for async-shared state (pixiv token cache), `AtomicBool` for feature gates.
|
||||||
- **Site adapter convention**: each site module exports `PATTERN: LazyLock<Regex>`, `enabled() -> bool`, `fetch_from_url(url) -> Result<Fetched, FetchError>`, plus `cache_key`/`is_retryable`/`media_headers`, and a unit struct `<Name>Site` implementing `site::Site`; the central dispatcher (`site/mod.rs`) only iterates the `SITES` registry. Adding a site = new `site/<name>/{mod.rs,interface.rs,model.rs}` + one `Box::new(...)` entry in `SITES` — the bot crate never lists sites (SetFormat whitelist, cache-key site lookup and startup validation all derive from the registry). Async trait methods return `SiteFuture` (a boxed `Pin<Box<dyn Future + Send>>`) because `async fn` in traits is not dyn-compatible.
|
- **Site adapter convention**: each site module exports `PATTERN: LazyLock<Regex>`, `enabled() -> bool`, `fetch_from_url(url) -> Result<Fetched, FetchError>`, plus `cache_key`/`is_retryable`/`media_headers`, and a unit struct `<Name>Site` implementing `site::Site`; the central dispatcher (`site/mod.rs`) only iterates the `SITES` registry. Adding a site = new `site/<name>/{mod.rs,interface.rs,model.rs}` + one `Box::new(...)` entry in `SITES` — the bot crate never lists sites (SetFormat whitelist, cache-key site lookup and startup validation all derive from the registry). Async trait methods return `SiteFuture` (a boxed `Pin<Box<dyn Future + Send>>`) because `async fn` in traits is not dyn-compatible.
|
||||||
- **Serde**: per-site `model.rs` are pure `Deserialize` DTOs mirroring API JSON; site structs in `interface.rs` have private fields, a `caption()` builder, and `impl From<SiteStruct> for Fetched`. Persisted payloads use internally-tagged enums (`#[serde(tag = "kind")]` / `type`).
|
- **Serde**: per-site `model.rs` are pure `Deserialize` DTOs mirroring API JSON; site structs in `interface.rs` have private fields, a `caption()` builder, and `impl From<SiteStruct> for Fetched`. Persisted payloads use internally-tagged enums (`#[serde(tag = "kind")]` / `type`).
|
||||||
- **Naming**: module-per-concern, snake_case files, `CamelCase` types, `snake_case` fns. `//!` module docs and `///` docs on non-obvious logic (syndication token, ugoira encoding, `display_text_range`).
|
- **Naming**: module-per-concern, snake_case files, `CamelCase` types, `snake_case` fns. `//!` module docs and `///` docs on non-obvious logic (syndication token, ugoira encoding, `display_text_range`).
|
||||||
- **Retries**: only `x-media::site::fetch` retries (3 attempts, `1 << attempt` backoff, HTTP errors only). Queue retries are explicit `QueueError::Retryable` with computed delay (`retry_delay_seconds`).
|
- **Retries**: only `x-media::site::fetch` retries (3 attempts, `1 << attempt` backoff, HTTP errors only); `site::fetch_once` is the same code path with a single attempt, used by inline queries whose answer window is shorter than the backoff. Queue retries are explicit `QueueError::Retryable` with computed delay (`retry_delay_seconds`).
|
||||||
- Logging via `log` macros (`pretty_env_logger`, level from `RUST_LOG`). Level convention: `info` = lifecycle + per-post business results (`sent`/`forwarded`/`copied`), admin/operator actions and anomalies (fallback, retry enqueue, dead-letter is `error`); `debug` = per-request detail (message/command/URL extraction, `fetching`/`fetched`, batch sends, queue processing, photo processing, inline queries). Full user-submitted URLs and message text only appear at `debug`; at `info` and above links are printed via the normalized cache key (`handlers::log_key`, e.g. `[key=twitter:123...]`) so logs stay short and do not echo user data.
|
- Logging via `log` macros (`pretty_env_logger`, level from `RUST_LOG`). Level convention: `info` = lifecycle + per-post business results (`sent`/`forwarded`/`copied`), admin/operator actions and anomalies (fallback, retry enqueue, dead-letter is `error`); `debug` = per-request detail (message/command/URL extraction, `fetching`/`fetched`, batch sends, queue processing, photo processing, inline queries). Full user-submitted URLs and message text only appear at `debug`; at `info` and above links are printed via the normalized cache key (`handlers::log_key`, e.g. `[key=twitter:123...]`) so logs stay short and do not echo user data.
|
||||||
|
|
||||||
## Important Files
|
## Important Files
|
||||||
@@ -72,15 +81,15 @@ Docker: `docker build -t tgxmb .` then `docker run --rm -d --name tgxmb --env-fi
|
|||||||
| File | Why it matters |
|
| File | Why it matters |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `crates/xmedia-bot/src/main.rs` | Startup sequence, webhook vs polling, graceful shutdown (SIGINT via teloxide ctrlc / SIGTERM via `stop_token` for docker, → sweep stop → admin msg → queue stop) |
|
| `crates/xmedia-bot/src/main.rs` | Startup sequence, webhook vs polling, graceful shutdown (SIGINT via teloxide ctrlc / SIGTERM via `stop_token` for docker, → sweep stop → admin msg → queue stop) |
|
||||||
| `crates/xmedia-bot/src/handlers/` | `statics.rs` = `CHAT_STORE`/`TASK_QUEUE`/`CONFIG` singletons (open `$DATA_DIR/task_queue.db`, default `data/` **relative to CWD**, dir auto-created); `commands.rs` = command dispatch (incl. the `/test <url>` parse-only debug command); `urls.rs` = URL extraction + retry enqueue; `inline.rs`/`callback.rs` = inline queries / edit-before-forward buttons |
|
| `crates/xmedia-bot/src/handlers/` | `statics.rs` = `CHAT_STORE`/`TASK_QUEUE`/`CONFIG` singletons (open `$DATA_DIR/task_queue.db`, default `data/` **relative to CWD**, dir auto-created); `commands.rs` = command dispatch (incl. `/test <url>` send-only, `/debug <url>` parse-only, the read-only `/settings` every chat member can read — unlike the admin-only `/bot_dict` raw dump — and template removal; `/start`/`/help` carry the guidance teloxide's `descriptions()` cannot render, and `/set_format` rejects unknown `{…}` placeholders, resetting with `-`); `urls.rs` = URL extraction + the per-URL pipeline (`url_media` takes a `PostSend` mode: chat settings vs `/test`'s suppressed actions); `inline.rs` = debounced inline queries (hotlink-protected and local media skipped); `callback.rs` = edit-before-forward buttons (dptree entry + testable `handle_callback` core, incl. `skip`) |
|
||||||
| `crates/xmedia-bot/src/send.rs` | Constants `MAX_MEDIA_GROUP = 9`; fallback chain; `classify_request_error`; download-and-reupload fallback triggered only by Telegram API errors (`is_media_fetch_failure` / `is_size_error`) |
|
| `crates/xmedia-bot/src/send/` | `mod.rs`: constants `MAX_MEDIA_GROUP = 10`; `classify_request_error`; the senders. `upload.rs`: download-and-reupload fallback triggered only by Telegram API errors (`is_media_fetch_failure` / `is_size_error`). `post_send.rs`: settlement (`settle_task`), cache write, post-send actions (dead-letter text via `failure_text`: post key + cause, since the raw error alone does not say which link died), queue handlers. `input_media.rs`: payload → `InputMedia` |
|
||||||
| `crates/xmedia-bot/src/photo.rs` | Pure-Rust photo processing (no ffmpeg): `png` (image-png) decode/encode + `zune-jpeg` decode + `fast_image_resize` Lanczos3 downscale + `jpeg-encoder`. Photos over Telegram's limits (width + height > 10000 px → `PHOTO_INVALID_DIMENSIONS`; bytes > 10 MiB) are decoded, downscaled keeping the format, PNG bit depth > 24 (RGBA 32-bit / 16-bit per channel) reduced to 24-bit RGB with alpha flattened white (≤24-bit untouched, never upconverted), and transcoded to JPEG only if still over the cap; memory budget guarded, otherwise the item's smaller fallback URL |
|
| `crates/xmedia-bot/src/photo.rs` | Pure-Rust photo processing (no ffmpeg): `png` (image-png) decode/encode + `zune-jpeg` decode + `fast_image_resize` Lanczos3 downscale + `jpeg-encoder`. Photos over Telegram's limits (width + height > 10000 px → `PHOTO_INVALID_DIMENSIONS`; bytes > 10 MiB) are decoded, downscaled keeping the format, PNG bit depth > 24 (RGBA 32-bit / 16-bit per channel) reduced to 24-bit RGB with alpha flattened white (≤24-bit untouched, never upconverted), and transcoded to JPEG only if still over the cap; memory budget guarded, otherwise the item's smaller fallback URL |
|
||||||
| `crates/x-media/src/site/mod.rs` | Dispatcher, `Fetched`/`FetchError`, shared `CLIENT`, `download_media` (adds `Referer: https://www.pixiv.net/` for `pximg.net` hotlink protection) |
|
| `crates/x-media/src/site/mod.rs` | Dispatcher, `Fetched`/`FetchError`, shared `CLIENT`, `download_media` (adds `Referer: https://www.pixiv.net/` for `pximg.net` hotlink protection), `needs_media_headers` (the same per-site rule, asked by the inline path to skip what Telegram cannot fetch) |
|
||||||
| `crates/x-media/src/site/pixiv/api.rs` | OAuth token exchange (hardcoded app client id/secret), access-token cache, ugoira zip→MP4 via ffmpeg in `spawn_blocking` |
|
| `crates/x-media/src/site/pixiv/api.rs` | OAuth token exchange (hardcoded app client id/secret), access-token cache, ugoira zip→MP4 via ffmpeg in `spawn_blocking` |
|
||||||
| `Dockerfile` | Multi-stage: cached dep layer via stub sources + `touch *.rs` mtime bump (cargo's freshness is mtime-based and `cargo clean -p` removes 0 files — the touch is what forces the real sources to rebuild while deps stay cached), static ffmpeg from ffmpeg.martin-riedl.de (`FFMPEG_URL` arg, optional `FFMPEG_SHA256` checksum, `unzip -t` integrity check), `debian:bookworm-slim` runtime, entrypoint. Runtime ships **no libssl/libcrypto/CA bundle** — rustls webpki-roots handles all TLS, and the static ffmpeg only processes local files (downloads go through reqwest) |
|
| `Dockerfile` | Multi-stage: cached dep layer via stub sources + `touch *.rs` mtime bump (cargo's freshness is mtime-based and `cargo clean -p` removes 0 files — the touch is what forces the real sources to rebuild while deps stay cached), static ffmpeg from ffmpeg.martin-riedl.de (`FFMPEG_URL` arg, optional `FFMPEG_SHA256` checksum, `unzip -t` integrity check), `debian:bookworm-slim` runtime, entrypoint. Runtime ships **no libssl/libcrypto/CA bundle** — rustls webpki-roots handles all TLS, and the static ffmpeg only processes local files (downloads go through reqwest) |
|
||||||
| `docker-entrypoint.sh` | Privilege drop: `useradd` with `LOCAL_USER_ID` (default 9001) + `setpriv` (no gosu on bookworm-slim) |
|
| `docker-entrypoint.sh` | Privilege drop: `useradd` with `LOCAL_USER_ID` (default 9001) + `setpriv` (no gosu on bookworm-slim) |
|
||||||
| `docker-compose.yml.example` | Deployment env reference (real `docker-compose.yml` is gitignored). Ships nginx-proxy + acme-companion: webhook mode needs TLS termination in front (teloxide's axum listener is HTTP-only; `WEBHOOK_CERT` only feeds `set_webhook`), bot exposes `VIRTUAL_HOST`/`VIRTUAL_PORT` on the shared `proxy` network, no host port; container names `nginx-proxy`/`acme-companion`/`tgxmb`, start order via `depends_on` (proxy → acme → bot) |
|
| `docker-compose.yml.example` | Deployment env reference (real `docker-compose.yml` is gitignored). Ships nginx-proxy + acme-companion: webhook mode needs TLS termination in front (teloxide's axum listener is HTTP-only; `WEBHOOK_CERT` only feeds `set_webhook`), bot exposes `VIRTUAL_HOST`/`VIRTUAL_PORT` on the shared `proxy` network, no host port; container names `nginx-proxy`/`acme-companion`/`tgxmb`, start order via `depends_on` (proxy → acme → bot) |
|
||||||
| `.github/workflows/docker.yml` | CI: build+push to Docker Hub on tag `v*`/master; **no test step**; buildx gha cache (`cache-from`/`cache-to`, scope `tgxmb-build`, `mode=max`) so cargo deps + ffmpeg layers are restored across runs |
|
| `.github/workflows/docker.yml` | CI: build+push to Docker Hub on tag `v*`/master, plus a build-only check on PRs touching the build inputs; **no test step**; verifies a release tag matches both crate versions; buildx gha cache (`cache-from` always, `cache-to` except on PRs, scope `tgxmb-build`, `mode=max`) so cargo deps + ffmpeg layers are restored across runs; `FFMPEG_URL`/`FFMPEG_SHA256` come from repo variables when set |
|
||||||
| `README.md` | Feature docs + command table (Chinese) |
|
| `README.md` | Feature docs + command table (Chinese) |
|
||||||
|
|
||||||
## Runtime/Tooling Preferences
|
## Runtime/Tooling Preferences
|
||||||
@@ -88,18 +97,18 @@ Docker: `docker build -t tgxmb .` then `docker run --rm -d --name tgxmb --env-fi
|
|||||||
- **Rust, stable, edition 2024**, workspace resolver 3. No `rust-version`/MSRV pin, no `rust-toolchain.toml` — recent stable is assumed. No nightly features.
|
- **Rust, stable, edition 2024**, workspace resolver 3. No `rust-version`/MSRV pin, no `rust-toolchain.toml` — recent stable is assumed. No nightly features.
|
||||||
- Package manager: **Cargo** (workspace with path dep `x-media` ← `xmedia-bot`). No `[workspace.package]`/shared deps — each crate lists deps independently.
|
- Package manager: **Cargo** (workspace with path dep `x-media` ← `xmedia-bot`). No `[workspace.package]`/shared deps — each crate lists deps independently.
|
||||||
- **TLS is rustls end-to-end** (no native-tls/openssl in the tree, no libssl in the Docker runtime image): `teloxide` is declared `default-features = false` with `["webhooks-axum", "macros", "rustls", "ctrlc_handler"]` (the removed `default` also carried `native-tls` and `ctrlc_handler` — the latter must stay); x-media's reqwest is `default-features = false` with `["json", "rustls-tls"]` (webpki-roots baked in, so the image ships no CA bundle). One reqwest 0.12.28 in the lock.
|
- **TLS is rustls end-to-end** (no native-tls/openssl in the tree, no libssl in the Docker runtime image): `teloxide` is declared `default-features = false` with `["webhooks-axum", "macros", "rustls", "ctrlc_handler"]` (the removed `default` also carried `native-tls` and `ctrlc_handler` — the latter must stay); x-media's reqwest is `default-features = false` with `["json", "rustls-tls"]` (webpki-roots baked in, so the image ships no CA bundle). One reqwest 0.12.28 in the lock.
|
||||||
- **Versioning**: bump the version in all three places (`crates/x-media/Cargo.toml`, `crates/xmedia-bot/Cargo.toml`, `Cargo.lock`) and **keep `README.md`, `README.en.md` and `AGENTS.md` in sync with the code on every bump**, then commit (`chore: bump version to X.Y.Z`), create an annotated tag `vX.Y.Z`, and push branch + tag (the tag push triggers the Docker Hub build).
|
- **Versioning**: bump the version in all three places (`crates/x-media/Cargo.toml`, `crates/xmedia-bot/Cargo.toml`, `Cargo.lock`) and **keep `README.md`, `README.en.md` and `AGENTS.md` in sync with the code on every bump**, then commit (`chore: bump version to X.Y.Z`), create an annotated tag `vX.Y.Z`, and push branch + tag (the tag push triggers the Docker Hub build). The tag must equal both crate versions: `.github/workflows/docker.yml` verifies that before building, and `--locked` verifies the lock file.
|
||||||
- Config is **environment-variable driven** (dotenv loads `.env`, gitignored; no `.env.example` exists). Key vars: `TELOXIDE_TOKEN` (required), `PIXIV_REFRESH_TOKEN`, `TWITTER_AUTH_TOKEN` (optional; x.com `auth_token` cookie — enables the logged-in GraphQL fallback that fetches NSFW tweets syndication withholds), `BOT_ADMIN` (comma-separated ids), `EDIT_MESSAGE_TTL_SECONDS` (default 86400), `LINK_CACHE_TTL_SECONDS` (default 604800), `DATA_DIR` (default `data`, CWD-relative; the SQLite dir, auto-created), `WEBHOOK`/`WEBHOOK_URL`/`WEBHOOK_LISTEN`/`WEBHOOK_PORT`/`WEBHOOK_CERT`/`WEBHOOK_SECRET_TOKEN` (webhook mode requires URL/listen/port, `.expect`ed; `WEBHOOK_CERT` is Telegram-facing self-signed validation only — TLS must be terminated by a reverse proxy), `RUST_LOG`, `TELOXIDE_PROXY`, `LOCAL_USER_ID` (entrypoint only).
|
- Config is **environment-variable driven** (dotenv loads `.env`, gitignored; no `.env.example` exists). Key vars: `TELOXIDE_TOKEN` (required), `PIXIV_REFRESH_TOKEN`, `TWITTER_AUTH_TOKEN` (optional; x.com `auth_token` cookie — enables the logged-in GraphQL fallback that fetches NSFW tweets syndication withholds), `BILIBILI_COOKIE` (optional; whole bilibili cookie string — bilibili dynamics fetch anonymously and add their own device cookies, this only rescues an egress IP that bilibili has hard-flagged with `-352`/412), `BOT_ADMIN` (comma-separated ids), `EDIT_MESSAGE_TTL_SECONDS` (default 86400), `LINK_CACHE_TTL_SECONDS` (default 604800), `CAPTION_QUOTE_TEXT_CHARS` (default 200; a post whose text — the `title` plus `content` joined, see `site::compose_text` — reaches this length gets that text wrapped in an expandable blockquote inside its caption, the URL and author line staying outside; `0` disables it. Applied at the send boundary in `send::quote_long_caption`, which locates the text as what follows the author link, so a `/set_format` that moves `{title}`/`{content}` elsewhere and pixiv's title-inside-a-link layout opt out; `copy_messages` forwards and queued retries inherit the wrap, while the edit-before-forward rewrite stays unquoted by design), `DATA_DIR` (default `data`, CWD-relative; the SQLite dir, auto-created), `WEBHOOK`/`WEBHOOK_URL`/`WEBHOOK_LISTEN`/`WEBHOOK_PORT`/`WEBHOOK_CERT`/`WEBHOOK_SECRET_TOKEN` (webhook mode requires URL/listen/port, `.expect`ed; `WEBHOOK_CERT` is Telegram-facing self-signed validation only — TLS must be terminated by a reverse proxy), `RUST_LOG`, `TELOXIDE_PROXY`, `LOCAL_USER_ID` (entrypoint only).
|
||||||
- SQLite via `rusqlite` with `bundled` feature (no system libsqlite needed). DB file `$DATA_DIR/task_queue.db` (default `data/task_queue.db`, CWD-relative — run from the workspace root, or `/app` in Docker; set `DATA_DIR` to pin state anywhere). Mount `./data` and `./cert` volumes.
|
- SQLite via `rusqlite` with `bundled` feature (no system libsqlite needed). DB file `$DATA_DIR/task_queue.db` (default `data/task_queue.db`, CWD-relative — run from the workspace root, or `/app` in Docker; set `DATA_DIR` to pin state anywhere). Mount `./data` and `./cert` volumes.
|
||||||
- `.gitattributes` enforces LF for `*.sh` (CRLF breaks shebangs in containers). `.gitignore`: `.env`, `data/`, `cert/`, `docker-compose.yml`, `/target`, `.idea/`.
|
- `.gitattributes` enforces LF for `*.sh` (CRLF breaks shebangs in containers). `.gitignore`: `.env`, `data/`, `cert/`, `docker-compose.yml`, `/target`, `.idea/`.
|
||||||
- Docs are in Chinese (README, AGENTS.md); user-facing bot strings are in English. Keep that split when editing user-facing strings and docs.
|
- Docs are in Chinese (README, AGENTS.md); user-facing bot strings are in English. Keep that split when editing user-facing strings and docs.
|
||||||
|
|
||||||
## Testing & QA
|
## Testing & QA
|
||||||
|
|
||||||
- **~125 tests, all inline `#[cfg(test)] mod tests`** — no `tests/` integration directories. Framework: built-in Rust test + `#[tokio::test]` (dev-deps only in `x-media`: tokio macros/rt-multi-thread, dotenv).
|
- **~200 tests, all inline `#[cfg(test)] mod tests`** — no `tests/` integration directories. Framework: built-in Rust test + `#[tokio::test]` (dev-deps only in `x-media`: tokio macros/rt-multi-thread, dotenv).
|
||||||
- No mocking framework anywhere (no mockito/wiremock/mockall). Conventions: pure-function units (regex parsing, serde round-trips, chunking, retry math) tested synchronously; async tests use real dependencies — file-backed SQLite via `tempfile` (`queue.rs::new_queue()` helper), live network fetches.
|
- No mocking framework anywhere (no mockito/wiremock/mockall). Conventions: pure-function units (regex parsing, serde round-trips, chunking, retry math) tested synchronously; async tests use real dependencies — file-backed SQLite via `tempfile` (`queue.rs::new_queue()` helper), live network fetches.
|
||||||
- Live-network tests exist in `site/twitter/interface.rs` (5), `site/bsky/interface.rs` (2), `site/misskey/interface.rs` (1), `site/pixiv/api.rs` (1); `photo.rs` adds one `#[ignore = "heavy: …"]` test. Test gating convention (enforced by `.github/workflows/ci.yml`): pure unit tests always run; live-network tests carry `#[ignore = "live network: ..."]` (run via `cargo test --workspace -- --ignored live`); token-gated pixiv tests early-return when `PIXIV_REFRESH_TOKEN` is absent **or empty** (an unset GitHub secret arrives as `""` — `is_err()` alone would run them tokenless and fail). Run the full offline suite with `cargo test --workspace`.
|
- Live-network tests exist in `site/twitter/interface.rs` (5), `site/bsky/interface.rs` (2), `site/misskey/interface.rs` (1), `site/bilibili/interface.rs` (4), `site/pixiv/api.rs` (1); `photo.rs` adds one `#[ignore = "heavy: …"]` test. `site/mod.rs` also has a **token-gated but not `#[ignore]`d** pixiv download test (`download_media_pixiv_original_with_referer`): it hits `i.pximg.net` whenever `PIXIV_REFRESH_TOKEN` is set, so a local `cargo test --workspace` is not fully offline and can flake on a pixiv CDN body timeout. `disabled_site_is_reported_not_ignored` (same file) is gated the other way round: it asserts `fetch` answers `FetchError::Disabled { site: "pixiv" }` for a pixiv link and early-returns when `PIXIV_REFRESH_TOKEN` **is** set (the site is then enabled). Test gating convention (enforced by `.github/workflows/ci.yml`): pure unit tests always run; live-network tests carry `#[ignore = "live network: ..."]` (run via `cargo test --workspace -- --ignored live`); token-gated pixiv tests early-return when `PIXIV_REFRESH_TOKEN` is absent **or empty** (an unset GitHub secret arrives as `""` — `is_err()` alone would run them tokenless and fail), and the bilibili live tests early-return when the API answers risk control (`-352`, which bilibili applies per IP by request volume). Run the full offline suite with `cargo test --workspace`.
|
||||||
- Fixtures are inline `serde_json::json!` builder fns (`fixture()`, `thread_json()`, `illust_json()`), not files. The shared `CLIENT` sets `pool_max_idle_per_host(0)` under `#[cfg(test)]` to avoid cross-runtime `DispatchGone`.
|
- Fixtures are inline `serde_json::json!` builder fns (`fixture()`, `thread_json()`, `illust_json()`), not files. The shared `CLIENT` sets `pool_max_idle_per_host(0)` under `#[cfg(test)]` to avoid cross-runtime `DispatchGone`.
|
||||||
- **CI** — `.github/workflows/ci.yml` runs `cargo fmt --check` + `cargo clippy --workspace --all-targets -- -D warnings` + `cargo test --workspace` + a `actions-rust-lang/audit` dependency-vulnerability gate (offline, no secrets, on every push/PR) and a `live` job (schedule/manual/tag only, `PIXIV_REFRESH_TOKEN`/`TWITTER_AUTH_TOKEN` from secrets, `continue-on-error`) for the `#[ignore]`d live + token tests. `.github/workflows/docker.yml` builds/pushes the image only.
|
- **CI** — `.github/workflows/ci.yml` (actions pinned to commit SHAs, `--locked` on every cargo invocation, `concurrency` cancels superseded runs, `RUST_BACKTRACE=1`) runs `cargo fmt --check` + `cargo clippy --workspace --all-targets --locked -- -D warnings` + `cargo test --workspace --locked` + a release-profile `cargo build --release --locked` + an `actions-rust-lang/audit` dependency-vulnerability gate (offline, no secrets, on every push/PR) and a `live` job (schedule/manual/tag only, `-p x-media` since every network/secret-gated test lives there, `continue-on-error`) for the `#[ignore]`d live + token tests. `.github/workflows/docker.yml` builds and pushes the image on master/tag and runs a **build-only check on pull requests touching the build inputs** (`Dockerfile`, entrypoint, manifests, `.dockerignore`); a release tag must match both crate versions or the build stops, and `FFMPEG_URL`/`FFMPEG_SHA256` are taken from repository variables when set (a release can pin an exact ffmpeg build). `.github/dependabot.yml` keeps crates, the pinned actions and the Docker base images current.
|
||||||
- Untested and hard to test without a mock seam: `handlers.rs` (depends directly on teloxide `Bot`); `main.rs`, `config.rs`, `state.rs`; `media.rs`, `lib.rs`, all `model.rs`.
|
- Untested and hard to test without a mock seam: `main.rs`, `config.rs`, `db.rs`, `handlers/statics.rs`, `media_sender.rs` (holds the `MockSender` itself); in `x-media`: `media.rs`, `lib.rs`, all `model.rs`. The `commands.rs` *executor* needs a real `Bot` (only its pure report builder is tested). Everything else — `handlers/{mod,callback,inline,urls}.rs`, `send/*`, `ctx.rs`, `state.rs`, `queue.rs`, `link_cache.rs`, `rate_limit.rs` — is driven through `TestStores`/`ctx::test_support` and the scripted `MockSender`.
|
||||||
- No coverage tracking.
|
- No coverage tracking.
|
||||||
|
|||||||
@@ -0,0 +1,168 @@
|
|||||||
|
# Bilibili 动态支持:研究与实现记录
|
||||||
|
|
||||||
|
状态:已实现(`crates/x-media/src/site/bilibili/`)。本文记录上游调研、实测数据与最终设计;
|
||||||
|
长期契约以 `AGENTS.md` 为准。
|
||||||
|
|
||||||
|
范围:**只发动态里的图片与动图**。动态内嵌视频不发流,降级为封面图;`b23.tv` 短链不匹配;
|
||||||
|
视频页 / 番剧 / 直播间 / 专栏 / 音频均不支持。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. 上游实现研究
|
||||||
|
|
||||||
|
### 1.1 nazurin(`nazurin/sites/bilibili/`,4 个文件 ~6 KB)
|
||||||
|
|
||||||
|
- 入口正则:`t\.bilibili\.com/(\d+)`、`t\.bilibili\.com/h5/dynamic/detail/(\d+)`、`bilibili\.com/opus/(\d+)`。
|
||||||
|
- 请求:`GET https://api.bilibili.com/x/polymer/web-dynamic/v1/detail?id={id}`,仅加 `Referer: https://t.bilibili.com/{id}`。
|
||||||
|
**无 cookie、无 WBI 签名、无 `build` 参数**。
|
||||||
|
- 错误:`code == 4101147` → not found;`code != 0` 或缺 `data` → 报错。
|
||||||
|
- 媒体:只取 `item.modules.module_dynamic.major.draw.items[].src`;缩略图 `src + "@518w.jpg"`;
|
||||||
|
`size` 字段单位是 **KB**。`major` 为空或 `draw.items` 为空 → "No image found"。
|
||||||
|
**忽略视频、转发(forward)与纯文字动态**。
|
||||||
|
- caption:`"#" + module_author.name` + `module_dynamic.desc.text`,链接写死 `https://www.bilibili.com/opus/{id}`。
|
||||||
|
|
||||||
|
### 1.2 telegram-bili-feed-helper(`biliparser/provider/bilibili/`,9 个文件 ~57 KB)
|
||||||
|
|
||||||
|
- 9 个策略类(Video/Opus/Live/Audio/Read + Feed 基类 + Credential + api 工具):门禁正则
|
||||||
|
`bilibili\.com|b23\.tv|BV\w{10}|av\d+`,再分流,兜底 `client.head(url)` 跟随重定向后按子串分流。
|
||||||
|
- 动态:`GET /x/polymer/web-dynamic/desktop/v1/detail?id={id}&build=11605`(**单条,无分页**);
|
||||||
|
客户端带桌面 UA、随机 `buvid3={uuid}infoc`;登录态用 `bilibili-api-python` 的 `Credential`
|
||||||
|
(Redis 持久化 `SESSDATA/bili_jct/buvid3/buvid4/ac_time_value/DedeUserID`,扫码登录)。
|
||||||
|
- **同样没有 WBI 签名 / appkey 签名**:playurl 用的是非 WBI 的 `/x/player/playurl`。
|
||||||
|
- 媒体:`major.type` 分派 —— DRAW 取全部 `items[].src`;ARCHIVE/PGC/ARTICLE/MUSIC/COMMON/LIVE
|
||||||
|
只取一张 `cover`;FORWARD 取原动态作者/正文并递归进 `orig` 找媒体。
|
||||||
|
- 视频:仅独立 video 策略解析(`qn` 720P→480P→360P 试 durl,再退 DASH + ffmpeg 合并);
|
||||||
|
**动态内嵌视频只发封面**。
|
||||||
|
- 错误:要求 `status==200 && code==0`;风控 `-352`/`-412` 无特殊处理。
|
||||||
|
|
||||||
|
### 1.3 取舍
|
||||||
|
|
||||||
|
| 维度 | nazurin | bff | 本仓库 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| 接口 | `v1/detail?id=` | `desktop/v1/detail?id=&build=` | `v1/detail?id=`(实测可用) |
|
||||||
|
| 认证 | 无 | buvid3 + SESSDATA | 默认匿名;可选 `BILIBILI_COOKIE` |
|
||||||
|
| WBI | 无 | 无 | 不实现(无需求) |
|
||||||
|
| 图片 | `major.draw.items` | 同 + forward 递归 | 同,加 `orig` 递归、`http→https`、`.gif → Animated` |
|
||||||
|
| 视频 | 完全忽略 | 动态内嵌视频发封面 | 发封面(不发流) |
|
||||||
|
| 短链 | 不匹配 | 跟随重定向 | 不匹配(多数短链是视频,会让"静默忽略"变成失败提示) |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. 实测验证(2026-09-17,真实请求)
|
||||||
|
|
||||||
|
| 验证项 | 结果 |
|
||||||
|
|---|---|
|
||||||
|
| `v1/detail?id=`(无 cookie、UA `Mozilla/5.0`、带 Referer) | `200 {"code":0}` ✅ |
|
||||||
|
| 同上,不带 cookie 也不带 Referer | `200 {"code":0}` ✅(无强制鉴权) |
|
||||||
|
| bff 的 `bilibili_pc/…Electron/22.3.27` UA | `code:-352` ❌ → **不要抄它的 UA** |
|
||||||
|
| `desktop/v1/detail?build=11605` | `code:-352` ❌ |
|
||||||
|
| `feed/space?host_mid=`(用户时间线) | 首次成功、随后 `-352`,也见过 HTTP 412 → **不碰** |
|
||||||
|
| 不存在 / 已删除的动态 | `code:500` "Cannot read property 'only_fans' of undefined"(nazurin 的 4101147 已失效) |
|
||||||
|
| 非数字 id | `code:-400` param parsing failed |
|
||||||
|
| 图片 `i0.hdslb.com/bfs/new_dyn/*.jpg` | `HEAD 200 image/jpeg`,带/不带 Referer 均可;`+@518w.jpg` → 25–42 KB ✅ |
|
||||||
|
| `t.bilibili.com/h5/dynamic/detail/<id>` | `200` ✅ |
|
||||||
|
| `m.bilibili.com/dynamic/<id>` | `302 → t.bilibili.com/<id>` ✅ |
|
||||||
|
| `www.bilibili.com/opus/<id>` | `200`,转发动态 `302 → t.bilibili.com/<id>` ✅ |
|
||||||
|
| `b23.tv/BV1JTtt6JEZu` | `302 → www.bilibili.com/video/BV…`(视频) |
|
||||||
|
| `b23.tv/<无效码>` | **HTTP 200** + `{"code":-404}` ⚠️ 短链判定不能只看状态码 |
|
||||||
|
| `playurl`(仅调研用,未采用) | `fnval=1` 匿名给 durl:720P=9.18 MiB / 360P=2.97 MiB;`fnval=4048` 匿名 DASH 上限仅 480P |
|
||||||
|
| `dyn_archive` 字段 | 有 `aid/bvid/cover/title/duration_text`,**没有 `cid`**(所以发流要再来一次 `view` 请求) |
|
||||||
|
| **风控阶梯(同一 IP 连续请求后实测)** | ① 无 cookie → `-352`;② 仅 `buvid3` → 仍 `-352`;③ `buvid3`+`buvid4`(取自匿名 `/x/frontend/finger/spi`)→ **`code:0` 恢复**;④ 继续高频请求后 → 连同 buvid 一起 `-352`(此时只有登录 cookie 或换 IP) |
|
||||||
|
| **正文位置(24 条真实动态逐条审计)** | 有正文的动态都在 `module_dynamic.desc.text`(图文/转发/纯文字,含 34–193 字样本);**AV(视频投稿)动态 `desc` 恒为 `null`**,内容在 `major.archive.title` / `.desc` 卡片里 → 已做 title 回退 |
|
||||||
|
| **`features=itemOpusStyle` 的效果** | 同一端点带此参数后,图文帖改为 `major.opus` 形态:`pics[]`(图,key 是 `url`)、`summary.text`(正文,未截断,实测 307 字整段)、`title`(可选标题);不带参数则是 legacy `major.draw` + `desc`,而 **opus 图文帖的 `desc` 为 `null`、正文与标题完全丢失**(`opus/1248857553488576532`:legacy `desc:null`,带参数 `summary.text="[doge_金箍]黑白搭配"`)。AV / 转发帖不受该参数影响 → 适配器改为请求时带参数,并保留 legacy 形态兜底 |
|
||||||
|
| feed 与 detail 的差异 | `feed/space` 的 item 会把 `desc.text` 挖空,**只有 detail 有正文** → 排查时不要用 feed 数据判断正文缺失 |
|
||||||
|
| 不存在的 19 位 id | `4101105 请求数据发生错误`(提示可重试,但只出现在不可能存在的 id 上)→ 仍归入永久错误,见 `code_error` 注释 |
|
||||||
|
|
||||||
|
测试样本(live 测试用):
|
||||||
|
|
||||||
|
| 样本 | id | 期望 |
|
||||||
|
|---|---|---|
|
||||||
|
| 图片动态(2 图 + 话题) | `1245284537985925159` | 2 个 `Illustration`,`{tags}` = `ALin出道20周年快乐` |
|
||||||
|
| 转发动态 | `1248982077447077907` | 媒体来自 `orig`(1 图),正文可含 `//@` |
|
||||||
|
| 视频动态 | `1248717597691609105` | 封面 1 张 `Illustration` |
|
||||||
|
| 纯文字动态 | `1246767523595026450` | `media` 为空 |
|
||||||
|
|
||||||
|
关键字段路径:
|
||||||
|
|
||||||
|
```
|
||||||
|
data.item.id_str
|
||||||
|
data.item.modules.module_author.{name,mid}
|
||||||
|
data.item.modules.module_dynamic.desc.text
|
||||||
|
data.item.modules.module_dynamic.topic.{id,name} # 单话题,{tags} 来源
|
||||||
|
data.item.modules.module_dynamic.major.{draw.items[].src, archive.cover}
|
||||||
|
data.item.orig # 转发时存在,结构与 item 相同
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. 实现
|
||||||
|
|
||||||
|
```
|
||||||
|
crates/x-media/src/site/bilibili/mod.rs # re-export
|
||||||
|
crates/x-media/src/site/bilibili/interface.rs # PATTERN / cache_key / enabled / is_retryable /
|
||||||
|
# media_headers / BilibiliSite / fetch / code_error /
|
||||||
|
# From<Item> for Fetched / caption / 12 单测 + 2 live
|
||||||
|
crates/x-media/src/site/bilibili/model.rs # 纯 Deserialize DTO(全 Option)
|
||||||
|
```
|
||||||
|
|
||||||
|
- **正则**(同时用于分发、抽 id、缓存键,一个正则三用):
|
||||||
|
`^(?:https?://)?(?:www|t|m)\.bilibili\.com/(?:opus/|dynamic/|h5/dynamic/detail/)?(\d+)`
|
||||||
|
- **缓存键**:`bilibili:<动态 id>`;`source_url` 统一 `https://www.bilibili.com/opus/{id}`。
|
||||||
|
- **请求**:`GET /x/polymer/web-dynamic/v1/detail?id=` + `Referer: https://www.bilibili.com/`;
|
||||||
|
`Cookie` 头按优先级取:`BILIBILI_COOKIE` → 缓存的设备 cookie(`GET /x/frontend/finger/spi` 取 `buvid3`/`buvid4`,
|
||||||
|
进程内缓存一次;取不到就不带 cookie,仅 debug 日志)→ 无。指纹接口本身失败**不**让抓取失败。
|
||||||
|
走共享 `CLIENT`(UA `Mozilla/5.0`,30s 超时,`TELOXIDE_PROXY` 透传)。
|
||||||
|
- **错误映射**:`0` → 成功;`-352/-412` 与 HTTP 412 → `Transient`(可重试,队列退避;首次记一条 warn 提示
|
||||||
|
`BILIBILI_COOKIE`);`500`/`4101147` → `NotFound`(永久);其他 code → `Site`(永久)。
|
||||||
|
- **媒体**:
|
||||||
|
- `major.opus.pics[]`(带 `features=itemOpusStyle` 时的图文帖形态,字段名是 `url`)→ 每张一张图;
|
||||||
|
其次 `major.draw.items[]`(legacy,字段名 `src`)→ 同样逐张;`http://` / `//` → `https://`,非 https 开头直接丢弃。
|
||||||
|
`.gif` → `Media::Animated`(`thumbnail_url` 留空,Telegram 自己取首帧——`@518w.jpg` 只对 jpg/webp 实测过),
|
||||||
|
其余 → `Media::Illustration`(`thumbnail_url = url + "@518w.jpg"`,兼作超大时的降级 URL)。
|
||||||
|
- `major.archive.cover` → 1 张 `Illustration`(视频不发流)。
|
||||||
|
- 转发且自身无媒体 → 递归取 `orig` 的媒体;正文拼 `//@{原作者}:\n{原文}`。
|
||||||
|
- 其他 major(PGC/ARTICLE/MUSIC/LIVE/COMMON)不建模 → 无媒体,走既有 "No media found"。
|
||||||
|
- **正文 / title**(按信息量从多到少回退):`major.opus.title` + `major.opus.summary.text`
|
||||||
|
→ `module_dynamic.desc.text` → `major.archive.title`。三者分别对应:图文文档(标题+正文)、
|
||||||
|
legacy/转发帖正文、视频投稿卡片标题。开头结尾空白做 trim;整体再由既有 `truncate_caption` 截断。
|
||||||
|
- **caption**(与 misskey 同形):`{opus 链接}\n<a href="space.bilibili.com/{mid}">{name}</a>: {正文}`;
|
||||||
|
`RenderData` 的 `{tags}` 来自话题名;正文由既有 `truncate_caption` 截断。
|
||||||
|
- **注册表**:`SITES` 末尾追加 → `/set_format` 白名单、链接缓存、启动校验、日志前缀全部自动生效。
|
||||||
|
- **bot 侧仅文案**:`handlers/commands.rs` 三处站点清单字符串 + `state.rs`/`handlers/mod.rs` 注释。
|
||||||
|
|
||||||
|
### 与原计划的偏差(及原因)
|
||||||
|
|
||||||
|
| 原计划 | 实际 | 原因 |
|
||||||
|
|---|---|---|
|
||||||
|
| `x/web-interface/view` + `playurl` 发视频 | 不做 | 需求收窄为图片/动图;视频只发封面 |
|
||||||
|
| `site/mod.rs` 加 `MAX_MEDIA_UPLOAD_BYTES` 常量 | 不加 | 没有视频尺寸决策就不需要该常量,避免跨 crate 耦合 |
|
||||||
|
| `b23.tv` 短链(跟随重定向) | 不匹配 | 多数短链指向视频,匹配后会把"静默忽略"变成用户的 "Failed to fetch media" |
|
||||||
|
| `validate()` 校验 cookie | 不做 | 匿名可用,cookie 失效不致命;校验要额外请求一个端点,收益低 |
|
||||||
|
| `media_headers` 给 hdslb 加 Referer | 返回 `None` | 实测图片与 durl 均无需 Referer(注释里记了这条验证) |
|
||||||
|
| 计划阶段认为设备 cookie 是 YAGNI,不实现 | **实现**(`buvid3`+`buvid4`) | 计划之后做了对照实验:同一 IP 上"无 cookie → -352、只有 buvid3 → -352、buvid3+buvid4 → code:0",说明这是对本适配器主要失败模式的直接修复,而不是冗余保险 |
|
||||||
|
| 只用不带参数的 `v1/detail` | 加 `features=itemOpusStyle` | 用户实测反馈"有内容的动态没有 title":不带参数时 opus 图文帖返回 legacy 形态,`desc` 为 `null`,正文与标题整个丢失。带参数后同一 ID 返回 `major.opus.summary.text` / `title` / `pics`。AV / 转发帖不受影响,legacy 形态仍保留为兜底 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. 测试与验证
|
||||||
|
|
||||||
|
- 单元(13):正则匹配/拒绝/忽略短链、缓存键归一、图片映射(https 归一 + 缩略图 + `.gif → Animated`)、
|
||||||
|
封面、转发取 `orig` 媒体与正文拼接、纯文字无媒体、caption 转义、业务 code 分类(可重试性)、URL 归一、
|
||||||
|
设备 cookie 拼装。
|
||||||
|
- live(3,`#[ignore = "live network: …"]`):设备 cookie 可取、图片动态 2 图、纯文字动态无媒体。
|
||||||
|
CI 的 `live` job 已覆盖。动态接口被风控时这两条 live 测试打印 `skipping:` 并提前返回(与 pixiv 的
|
||||||
|
token 门控同款约定),设备 cookie 那条仍会真实执行。
|
||||||
|
- 实测命令:
|
||||||
|
`cargo run -p x-media --example fetch -- https://www.bilibili.com/opus/1245284537985925159`
|
||||||
|
(输出 2 张 `https://i0.hdslb.com/…jpg` + `@518w.jpg` 缩略图 + 话题 tags)。
|
||||||
|
- 全套:`cargo fmt --check`、`cargo clippy --workspace --all-targets -- -D warnings`、`cargo test --workspace` 全绿。
|
||||||
|
|
||||||
|
## 5. 已知限制
|
||||||
|
|
||||||
|
- 风控按 IP/请求量漂移,阶梯见 §2 最后一行:轻度靠设备 cookie 自愈,重度需 `BILIBILI_COOKIE` 或换 IP。
|
||||||
|
被拦时按**可重试**失败处理(队列退避)+ 一条 warn,不会静默丢帖。
|
||||||
|
- 接口 schema 会漂移(`module_dynamic.major` 实测可为 `null` 而正文留在 `desc`);DTO 全 `Option`,
|
||||||
|
未知形态降级为"无媒体",不 panic。
|
||||||
|
- 动态内嵌视频只发封面图(与 bff 同策略),不下载流。
|
||||||
|
- 纯文字动态复用既有 "No media found" 回复。
|
||||||
|
- `b23.tv` 短链不被匹配(见上表)。
|
||||||
Generated
+587
-714
File diff suppressed because it is too large
Load Diff
+4
-2
@@ -11,6 +11,8 @@ ARG APP_NAME=telegram-twitter-media-bot
|
|||||||
# runners. `/redirect/latest/` floats to the newest release build; each build
|
# runners. `/redirect/latest/` floats to the newest release build; each build
|
||||||
# also ships a .sha256. Swap `amd64` for `arm64` when building arm64 images.
|
# also ships a .sha256. Swap `amd64` for `arm64` when building arm64 images.
|
||||||
ARG FFMPEG_URL=https://ffmpeg.martin-riedl.de/redirect/latest/linux/amd64/release/ffmpeg.zip
|
ARG FFMPEG_URL=https://ffmpeg.martin-riedl.de/redirect/latest/linux/amd64/release/ffmpeg.zip
|
||||||
|
# Arm64 images need this URL swapped for the `linux/arm64` build (currently
|
||||||
|
# hardcoded amd64; the workflow builds amd64 only — see docker.yml).
|
||||||
# Optional sha256 of ffmpeg.zip (pinned releases only): set to verify the
|
# Optional sha256 of ffmpeg.zip (pinned releases only): set to verify the
|
||||||
# download. The mirror publishes .sha256 sidecars next to pinned builds, e.g.
|
# download. The mirror publishes .sha256 sidecars next to pinned builds, e.g.
|
||||||
# https://ffmpeg.martin-riedl.de/download/linux/amd64/<id>_9.0/ffmpeg.zip.sha256
|
# https://ffmpeg.martin-riedl.de/download/linux/amd64/<id>_9.0/ffmpeg.zip.sha256
|
||||||
@@ -28,7 +30,7 @@ COPY crates/xmedia-bot/Cargo.toml crates/xmedia-bot/Cargo.toml
|
|||||||
RUN mkdir -p crates/x-media/src crates/xmedia-bot/src \
|
RUN mkdir -p crates/x-media/src crates/xmedia-bot/src \
|
||||||
&& printf 'fn main() {}\n' > crates/xmedia-bot/src/main.rs \
|
&& printf 'fn main() {}\n' > crates/xmedia-bot/src/main.rs \
|
||||||
&& : > crates/x-media/src/lib.rs \
|
&& : > crates/x-media/src/lib.rs \
|
||||||
&& cargo build --release -p xmedia-bot
|
&& cargo build --release --locked -p xmedia-bot
|
||||||
|
|
||||||
# 2. Static ffmpeg next (cached unless FFMPEG_URL changes), so source edits
|
# 2. Static ffmpeg next (cached unless FFMPEG_URL changes), so source edits
|
||||||
# never re-download it. The zip contains a single `ffmpeg` binary at the
|
# never re-download it. The zip contains a single `ffmpeg` binary at the
|
||||||
@@ -51,7 +53,7 @@ RUN wget -q -O /tmp/ffmpeg.zip "$FFMPEG_URL" \
|
|||||||
# removes 0 files and the stub binary silently ships.)
|
# removes 0 files and the stub binary silently ships.)
|
||||||
COPY crates/ ./crates/
|
COPY crates/ ./crates/
|
||||||
RUN find crates -type f -name '*.rs' -exec touch {} + \
|
RUN find crates -type f -name '*.rs' -exec touch {} + \
|
||||||
&& cargo build --release -p xmedia-bot
|
&& cargo build --release --locked -p xmedia-bot
|
||||||
|
|
||||||
# ---------- runtime stage ----------
|
# ---------- runtime stage ----------
|
||||||
FROM debian:bookworm-slim
|
FROM debian:bookworm-slim
|
||||||
|
|||||||
+26
-15
@@ -1,14 +1,18 @@
|
|||||||
# TelegramXMediaBot
|
# TelegramXMediaBot
|
||||||
|
|
||||||
A Telegram bot that turns post links from X / Twitter, Pixiv, Bluesky, and Misskey (misskey.io) into media messages (images, video, GIF) with the post's title, author, and tags.
|
A Telegram bot that turns post links from X / Twitter, Pixiv, Bluesky, Misskey (misskey.io), and Bilibili dynamics into media messages (images, video, GIF) with the post's title, author, and tags.
|
||||||
|
|
||||||
## Features
|
## Features
|
||||||
|
|
||||||
- Sending a link in a private chat fetches and sends the images, videos and GIFs automatically; oversized media is split into batches
|
- Sending a link in a private chat fetches and sends the images, videos and GIFs automatically; oversized media is split into batches (10 items per group)
|
||||||
- Text-only posts report "no media"; unsupported links are silently ignored
|
- Text-only posts report "no media"; unsupported links are silently ignored. Fetch failures name the reason (post gone / content withheld / source risk control / site not enabled)
|
||||||
- Inline queries (`@bot <link>`)
|
- Long posts (text ≥ `CAPTION_QUOTE_TEXT_CHARS`, default 200) show **the text part** of their caption inside a collapsible blockquote, with the link and author line left outside it
|
||||||
- Bind a forward channel for automatic forwarding; edit the caption before forwarding and apply custom templates
|
- Inline queries (`@bot <link>`) — except Pixiv images and locally transcoded animations, which Telegram cannot fetch (no Referer) and would show broken, so they are skipped; a supported link posted in a group gets a one-line hint to use the private chat or inline mode (channels stay silent)
|
||||||
- Failed sends are retried automatically with persistence; the user is notified after retries are exhausted
|
- `/start` explains the supported sites and how to use it; `/help` lists the commands plus argument syntax, the caption placeholders and the private-chat rule; the bot's profile description texts are set at startup
|
||||||
|
- `/settings` shows this chat's configuration (forward channel, edit-before-forward, per-site caption formats, saved templates); templates are added with `/set_template` and removed with `/remove_template`
|
||||||
|
- Bind a forward channel for automatic forwarding; edit the caption before forwarding and apply custom templates (the prompt carries Confirm / Skip buttons, states its expiry, and is marked expired in place once it lapses)
|
||||||
|
- Failed sends are retried automatically with persistence; the notice names which link failed, how long the retry waits, or the final cause
|
||||||
|
- The chat action stays on screen for the whole fetch, so long jobs (ugoira transcode, large uploads) do not look stalled
|
||||||
- Pixiv ugoira animations are transcoded to MP4; Bluesky videos are remuxed (HLS stream → MP4)
|
- Pixiv ugoira animations are transcoded to MP4; Bluesky videos are remuxed (HLS stream → MP4)
|
||||||
- Photos exceeding Telegram's size/dimension limits are compressed automatically (original format kept, JPEG fallback only when needed)
|
- Photos exceeding Telegram's size/dimension limits are compressed automatically (original format kept, JPEG fallback only when needed)
|
||||||
- Link-result cache: after a successful send the Telegram file ids and caption fields are cached locally, so a repeated link is re-sent from local state — no source-site request, no media file stored (expiry controlled by `LINK_CACHE_TTL_SECONDS`, default 7 days)
|
- Link-result cache: after a successful send the Telegram file ids and caption fields are cached locally, so a repeated link is re-sent from local state — no source-site request, no media file stored (expiry controlled by `LINK_CACHE_TTL_SECONDS`, default 7 days)
|
||||||
@@ -30,9 +34,11 @@ docker build -t tgxmb .
|
|||||||
docker run --rm -d --name tgxmb --env-file .env -v ./data:/app/data tgxmb
|
docker run --rm -d --name tgxmb --env-file .env -v ./data:/app/data tgxmb
|
||||||
```
|
```
|
||||||
|
|
||||||
Environment variables: `TELOXIDE_TOKEN` (required), `PIXIV_REFRESH_TOKEN`, `BOT_ADMIN`, `EDIT_MESSAGE_TTL_SECONDS`, `LINK_CACHE_TTL_SECONDS`, `RUST_LOG`, `TELOXIDE_PROXY`, `WEBHOOK*`, `TWITTER_AUTH_TOKEN` (optional).
|
Environment variables: `TELOXIDE_TOKEN` (required), `PIXIV_REFRESH_TOKEN`, `BOT_ADMIN`, `EDIT_MESSAGE_TTL_SECONDS`, `LINK_CACHE_TTL_SECONDS`, `RUST_LOG`, `TELOXIDE_PROXY`, `WEBHOOK*`, `TWITTER_AUTH_TOKEN` (optional), `BILIBILI_COOKIE` (optional).
|
||||||
|
|
||||||
NSFW tweets: the public syndication endpoint does not return sensitive content. Setting `TWITTER_AUTH_TOKEN` (the `auth_token` cookie value of a logged-in x.com session) lets the bot fetch NSFW media in the logged-in state only when it hits a withheld tweet; without it, the bot reports no media.
|
NSFW tweets: the public syndication endpoint does not return sensitive content. Setting `TWITTER_AUTH_TOKEN` (the `auth_token` cookie value of a logged-in x.com session) lets the bot fetch NSFW media in the logged-in state only when it hits a withheld tweet; without it the bot answers that the post's media is withheld and needs `TWITTER_AUTH_TOKEN`.
|
||||||
|
|
||||||
|
Bilibili dynamics are fetched anonymously by default (no login; the bot fetches bilibili's anonymous `buvid3`/`buvid4` device cookies itself to raise the success rate). If the server's egress IP gets hard-flagged by bilibili (persistent `risk control (-352)` log lines or HTTP 412), set `BILIBILI_COOKIE` (the whole cookie string from a logged-in browser, e.g. `SESSDATA=…; bili_jct=…`) to restore access. Only a dynamic's images and animations are sent; an attached video degrades to its cover image.
|
||||||
|
|
||||||
### Webhook deployment (needs a reverse proxy)
|
### Webhook deployment (needs a reverse proxy)
|
||||||
|
|
||||||
@@ -80,10 +86,12 @@ Telegram only accepts ports 443/80/88/8443.
|
|||||||
| Variable | Description |
|
| Variable | Description |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `TELOXIDE_TOKEN` | Bot token (required) |
|
| `TELOXIDE_TOKEN` | Bot token (required) |
|
||||||
| `PIXIV_REFRESH_TOKEN` | Pixiv refresh token; Pixiv is disabled without it |
|
| `PIXIV_REFRESH_TOKEN` | Pixiv refresh token; Pixiv is disabled without it (a pixiv link then gets an explicit "site not enabled" reply instead of silence) |
|
||||||
|
| `BILIBILI_COOKIE` | Optional bilibili cookie string (`SESSDATA=…; bili_jct=…`); only needed when the egress IP stays risk-controlled (device cookies are fetched automatically) |
|
||||||
| `BOT_ADMIN` | Admin chat IDs, comma-separated; receives start/stop notifications |
|
| `BOT_ADMIN` | Admin chat IDs, comma-separated; receives start/stop notifications |
|
||||||
| `EDIT_MESSAGE_TTL_SECONDS` | Edit-before-forward record expiry in seconds, default 86400 |
|
| `EDIT_MESSAGE_TTL_SECONDS` | Edit-before-forward record expiry in seconds, default 86400; once lapsed the prompt is rewritten in place to "expired — nothing was forwarded" (no extra message) |
|
||||||
| `LINK_CACHE_TTL_SECONDS` | Link-result cache expiry in seconds, default 604800 (7 days) |
|
| `LINK_CACHE_TTL_SECONDS` | Link-result cache expiry in seconds, default 604800 (7 days) |
|
||||||
|
| `CAPTION_QUOTE_TEXT_CHARS` | **The text part** of the caption (the joined `{title}` + `{content}`) is wrapped in a collapsible blockquote once it reaches this many characters, default 200; `0` disables |
|
||||||
| `DATA_DIR` | Data directory (where the SQLite `task_queue.db` lives), default `data` (relative to the working directory, created automatically) |
|
| `DATA_DIR` | Data directory (where the SQLite `task_queue.db` lives), default `data` (relative to the working directory, created automatically) |
|
||||||
| `RUST_LOG` | Log level |
|
| `RUST_LOG` | Log level |
|
||||||
| `TELOXIDE_PROXY` | HTTP proxy (e.g. `http://127.0.0.1:10808`); applies to both the Telegram Bot API and site fetches — required on restricted networks (e.g. behind the GFW) |
|
| `TELOXIDE_PROXY` | HTTP proxy (e.g. `http://127.0.0.1:10808`); applies to both the Telegram Bot API and site fetches — required on restricted networks (e.g. behind the GFW) |
|
||||||
@@ -109,14 +117,17 @@ Telegram only accepts ports 443/80/88/8443.
|
|||||||
| `/help` | List all commands and usage (this command table) |
|
| `/help` | List all commands and usage (this command table) |
|
||||||
| `/set_forward_channel <channel>` | Set the forward channel: `@channel` or channel ID; media messages are forwarded to it automatically afterwards |
|
| `/set_forward_channel <channel>` | Set the forward channel: `@channel` or channel ID; media messages are forwarded to it automatically afterwards |
|
||||||
| `/remove_forward_channel` | Remove the forward channel |
|
| `/remove_forward_channel` | Remove the forward channel |
|
||||||
| `/edit_before_forward` | Toggle "edit before forward": when enabled, the bot posts a prompt after forwarding; replying to it edits the first forwarded message's caption (or taps a template button to apply one) |
|
| `/edit_before_forward` | Toggle "edit before forward": when enabled, the bot posts a prompt after forwarding; replying to it edits the first forwarded message's caption (or tapping a template button applies one), then `↩️ Confirm` forwards and `🛑 Skip` drops this forward; the prompt states its expiry and is marked expired in place when it lapses (nothing is forwarded) |
|
||||||
| `/set_template <name>` | Reply to a message containing `[]` to save it as a named template; `[]` is replaced by the original post link when forwarding (used with "edit before forward") |
|
| `/set_template <name>` | Reply to a message containing `[]` to save it as a named template; `[]` is replaced by the original post link when forwarding (used with "edit before forward") |
|
||||||
| `/set_format <site> <format>` | Customize the caption format for one site. Sites: `twitter` / `bsky` / `pixiv` / `misskey`. Placeholders: `{url}` `{author}` `{author_url}` `{title}` `{tags}` |
|
| `/remove_template <name>` | Remove a template (names are listed by `/settings`; the prompt's keyboard shows at most 60) |
|
||||||
|
| `/settings` | Show this chat's configuration: forward channel, edit-before-forward, per-site caption formats, saved templates |
|
||||||
|
| `/set_format <site> <format>` | Customize the caption format for one site. Sites: `twitter` / `bsky` / `pixiv` / `misskey` / `bilibili`. Placeholders: `{url}` `{author}` `{author_url}` `{title}` `{content}` `{tags}`; unknown placeholders are rejected with the list of valid ones, and `-` restores the site's built-in format (preview with `/debug <link>`) |
|
||||||
| `/clear_cache [link]` | Clear the link cache (admin only); with a link only that entry, otherwise everything |
|
| `/clear_cache [link]` | Clear the link cache (admin only); with a link only that entry, otherwise everything |
|
||||||
| `/bot_dict` | Show the current chat state (debugging) |
|
| `/bot_dict` | Show the current chat state (debugging; admin only) |
|
||||||
| `/test <link>` | Debug: parse a link and report the parse result only (site, title, author, tags, media list) — no media is sent |
|
| `/test <link>` | Parse a link and send its media; no channel forward, no edit-before-forward prompt (send only) |
|
||||||
|
| `/debug <link>` | Debug: parse a link and report the parse result only (site, title, author, tags, media list) — no media is sent |
|
||||||
|
|
||||||
Link processing works only in private chats; commands work in any chat.
|
Link processing works only in private chats; commands work in any chat. A supported link posted in a group gets a one-line hint to use the private chat or inline mode; channels stay silent.
|
||||||
|
|
||||||
## Notes
|
## Notes
|
||||||
|
|
||||||
|
|||||||
@@ -1,14 +1,18 @@
|
|||||||
# TelegramXMediaBot
|
# TelegramXMediaBot
|
||||||
|
|
||||||
Telegram 机器人,将 X / Twitter、Pixiv、Bluesky、Misskey (misskey.io) 的帖子链接转换为媒体消息发送,附带帖子标题、作者与标签。
|
Telegram 机器人,将 X / Twitter、Pixiv、Bluesky、Misskey (misskey.io)、Bilibili 动态的帖子链接转换为媒体消息发送,附带帖子标题、作者与标签。
|
||||||
|
|
||||||
## 功能
|
## 功能
|
||||||
|
|
||||||
- 私聊发送链接后自动抓取并发送图片、视频与 GIF,超量图片自动分批
|
- 私聊发送链接后自动抓取并发送图片、视频与 GIF,超量图片自动分批(每批 10 张)
|
||||||
- 纯文字帖提示无媒体;不支持的链接静默忽略
|
- 纯文字帖提示无媒体;不支持的链接静默忽略。抓取失败会按原因分别提示(帖子已删除 / 内容受限 / 源站风控 / 站点未启用)
|
||||||
- 支持内联查询(`@机器人 <链接>`)
|
- 长帖(正文 ≥ `CAPTION_QUOTE_TEXT_CHARS`,默认 200)的**正文部分**用可折叠引用块展示,链接与作者行留在引用块外
|
||||||
- 可绑定转发频道自动转发;支持转发前编辑 caption 与自定义模板
|
- 支持内联查询(`@机器人 <链接>`;Pixiv 图片与本地转码的动图不支持内联 —— Telegram 取图时无法携带 Referer,会显示破图,因此跳过);在群聊里发链接会提示改用私聊或内联查询(频道内保持静默)
|
||||||
- 发送失败自动重试并持久化,重试耗尽后通知用户
|
- `/start` 说明支持的站点与用法,`/help` 列出命令、参数格式、caption 占位符与私聊限制;bot 资料页(description / short description)启动时一并设置
|
||||||
|
- `/settings` 查看本聊天配置(转发频道、转发前编辑开关、各站点 caption 格式、模板列表);模板可用 `/set_template` 增、`/remove_template` 删
|
||||||
|
- 可绑定转发频道自动转发;支持转发前编辑 caption 与自定义模板(提示消息带 Confirm / Skip 按钮并写明过期时间,过期后就地标记为已过期)
|
||||||
|
- 发送失败自动重试并持久化,重试耗尽后通知用户;提示会写明是哪条链接、重试等待多久、或最终失败的原因
|
||||||
|
- 抓取期间持续显示"正在输入 / 正在发送"状态,长任务(ugoira 转码、大图上传)不会看起来卡死
|
||||||
- Pixiv ugoira 动图自动转码为 MP4;Bluesky 视频自动转码(HLS 流 → MP4)
|
- Pixiv ugoira 动图自动转码为 MP4;Bluesky 视频自动转码(HLS 流 → MP4)
|
||||||
- 超过 Telegram 尺寸/大小限制的图片自动压缩(保持原格式,必要时转 JPEG)
|
- 超过 Telegram 尺寸/大小限制的图片自动压缩(保持原格式,必要时转 JPEG)
|
||||||
- 链接结果本地缓存:成功发送后缓存 Telegram file id 与 caption 等,再次收到相同链接直接本地重发,不再请求源站、不保存媒体文件(`LINK_CACHE_TTL_SECONDS` 控制过期,默认 7 天)
|
- 链接结果本地缓存:成功发送后缓存 Telegram file id 与 caption 等,再次收到相同链接直接本地重发,不再请求源站、不保存媒体文件(`LINK_CACHE_TTL_SECONDS` 控制过期,默认 7 天)
|
||||||
@@ -30,9 +34,11 @@ docker build -t tgxmb .
|
|||||||
docker run --rm -d --name tgxmb --env-file .env -v ./data:/app/data tgxmb
|
docker run --rm -d --name tgxmb --env-file .env -v ./data:/app/data tgxmb
|
||||||
```
|
```
|
||||||
|
|
||||||
环境变量:`TELOXIDE_TOKEN`(必填)、`PIXIV_REFRESH_TOKEN`、`BOT_ADMIN`、`EDIT_MESSAGE_TTL_SECONDS`、`LINK_CACHE_TTL_SECONDS`、`RUST_LOG`、`TELOXIDE_PROXY`、`WEBHOOK*`、`TWITTER_AUTH_TOKEN`(可选)。
|
环境变量:`TELOXIDE_TOKEN`(必填)、`PIXIV_REFRESH_TOKEN`、`BOT_ADMIN`、`EDIT_MESSAGE_TTL_SECONDS`、`LINK_CACHE_TTL_SECONDS`、`RUST_LOG`、`TELOXIDE_PROXY`、`WEBHOOK*`、`TWITTER_AUTH_TOKEN`(可选)、`BILIBILI_COOKIE`(可选)。
|
||||||
|
|
||||||
NSFW 推文:公开的 syndication 接口不返回敏感内容。设置 `TWITTER_AUTH_TOKEN`(登录 x.com 后浏览器 Cookie 里的 `auth_token` 值)后,bot 会仅在遇到 NSFW 推文时以登录态获取媒体;未设置则提示无媒体。
|
NSFW 推文:公开的 syndication 接口不返回敏感内容。设置 `TWITTER_AUTH_TOKEN`(登录 x.com 后浏览器 Cookie 里的 `auth_token` 值)后,bot 会仅在遇到 NSFW 推文时以登录态获取媒体;未设置则回复该推文内容受限(需要配置 `TWITTER_AUTH_TOKEN`)。
|
||||||
|
|
||||||
|
Bilibili 动态默认匿名抓取(无需登录,bot 会自动从 B 站的匿名指纹接口取 `buvid3`/`buvid4` 设备 cookie 以提高成功率)。若服务器出口 IP 被 B 站重度风控(日志里的 `risk control (-352)` 或 HTTP 412,且持续出现),设置 `BILIBILI_COOKIE`(登录后浏览器里整条 Cookie 串,如 `SESSDATA=…; bili_jct=…`)可恢复访问。当前只发送动态里的图片与动图,动态内嵌视频发送其封面。
|
||||||
|
|
||||||
### Webhook 部署(需要反向代理)
|
### Webhook 部署(需要反向代理)
|
||||||
|
|
||||||
@@ -80,10 +86,12 @@ Telegram 只接受 443/80/88/8443 端口。
|
|||||||
| 变量 | 说明 |
|
| 变量 | 说明 |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `TELOXIDE_TOKEN` | Bot token(必填) |
|
| `TELOXIDE_TOKEN` | Bot token(必填) |
|
||||||
| `PIXIV_REFRESH_TOKEN` | Pixiv 刷新令牌;未设置则禁用 Pixiv |
|
| `PIXIV_REFRESH_TOKEN` | Pixiv 刷新令牌;未设置则禁用 Pixiv(此时收到 pixiv 链接会明确回复「站点未启用」,不会静默忽略) |
|
||||||
|
| `BILIBILI_COOKIE` | 可选的 B 站 Cookie 串(`SESSDATA=…; bili_jct=…`),仅在出口 IP 被持续风控时才需要(设备 cookie 由 bot 自动获取) |
|
||||||
| `BOT_ADMIN` | 管理员聊天 ID,逗号分隔;接收启动/停止通知 |
|
| `BOT_ADMIN` | 管理员聊天 ID,逗号分隔;接收启动/停止通知 |
|
||||||
| `EDIT_MESSAGE_TTL_SECONDS` | 转发前编辑记录过期秒数,默认 86400 |
|
| `EDIT_MESSAGE_TTL_SECONDS` | 转发前编辑记录过期秒数,默认 86400;过期后提示消息会被就地改写为「已过期,未转发」(不额外发消息打扰) |
|
||||||
| `LINK_CACHE_TTL_SECONDS` | 链接结果缓存过期秒数,默认 604800(7 天) |
|
| `LINK_CACHE_TTL_SECONDS` | 链接结果缓存过期秒数,默认 604800(7 天) |
|
||||||
|
| `CAPTION_QUOTE_TEXT_CHARS` | 正文(`{title}` + `{content}` 合计)达到该长度(字符)时,caption 的**正文部分**用可折叠引用块包裹,默认 200;`0` 关闭 |
|
||||||
| `DATA_DIR` | 数据目录(SQLite 数据库 `task_queue.db` 所在目录),默认 `data`(相对工作目录,会自动创建) |
|
| `DATA_DIR` | 数据目录(SQLite 数据库 `task_queue.db` 所在目录),默认 `data`(相对工作目录,会自动创建) |
|
||||||
| `RUST_LOG` | 日志级别 |
|
| `RUST_LOG` | 日志级别 |
|
||||||
| `TELOXIDE_PROXY` | HTTP 代理(如 `http://127.0.0.1:10808`);同时作用于 Telegram Bot API 与站点抓取请求,网络受限环境(如 GFW)必需 |
|
| `TELOXIDE_PROXY` | HTTP 代理(如 `http://127.0.0.1:10808`);同时作用于 Telegram Bot API 与站点抓取请求,网络受限环境(如 GFW)必需 |
|
||||||
@@ -109,14 +117,17 @@ Telegram 只接受 443/80/88/8443 端口。
|
|||||||
| `/help` | 查看全部命令及用法(即本文档的命令表) |
|
| `/help` | 查看全部命令及用法(即本文档的命令表) |
|
||||||
| `/set_forward_channel <频道>` | 设置转发频道,参数为 `@频道名` 或频道 ID;设置后发送的媒体消息会自动转发到该频道 |
|
| `/set_forward_channel <频道>` | 设置转发频道,参数为 `@频道名` 或频道 ID;设置后发送的媒体消息会自动转发到该频道 |
|
||||||
| `/remove_forward_channel` | 取消转发频道 |
|
| `/remove_forward_channel` | 取消转发频道 |
|
||||||
| `/edit_before_forward` | 开关「转发前编辑」:开启后,转发成功后 bot 会发一条提示消息,回复它可修改第一条转发消息的 caption(或点击模板按钮套用模板) |
|
| `/edit_before_forward` | 开关「转发前编辑」:开启后,转发成功后 bot 会发一条提示消息,回复它可修改第一条转发消息的 caption(或点击模板按钮套用模板),再点 `↩️ Confirm` 才会真正转发,`🛑 Skip` 放弃本次转发;提示消息写明过期时间,过期后原地标记为已过期且不会转发 |
|
||||||
| `/set_template <名称>` | 回复一条含 `[]` 的消息,将其保存为命名模板;转发时 `[]` 会被替换为原帖链接(配合「转发前编辑」使用) |
|
| `/set_template <名称>` | 回复一条含 `[]` 的消息,将其保存为命名模板;转发时 `[]` 会被替换为原帖链接(配合「转发前编辑」使用) |
|
||||||
| `/set_format <站点> <格式>` | 自定义某站点的 caption 格式。站点:`twitter` / `bsky` / `pixiv` / `misskey`。占位符:`{url}` `{author}` `{author_url}` `{title}` `{tags}` |
|
| `/remove_template <名称>` | 删除某个模板(名称见 `/settings`;提示消息的模板按钮最多显示 60 个) |
|
||||||
|
| `/settings` | 查看本聊天配置:转发频道、转发前编辑开关、各站点 caption 格式、模板列表 |
|
||||||
|
| `/set_format <站点> <格式>` | 自定义某站点的 caption 格式。站点:`twitter` / `bsky` / `pixiv` / `misskey` / `bilibili`。占位符:`{url}` `{author}` `{author_url}` `{title}` `{content}` `{tags}`;未识别的占位符会被拒绝并列出可用项,格式填 `-` 恢复站点默认格式(可用 `/debug <链接>` 预览效果) |
|
||||||
| `/clear_cache [链接]` | 清空链接缓存(仅管理员);带链接只清该条,否则清空全部 |
|
| `/clear_cache [链接]` | 清空链接缓存(仅管理员);带链接只清该条,否则清空全部 |
|
||||||
| `/bot_dict` | 查看当前聊天状态(调试用) |
|
| `/bot_dict` | 查看当前聊天状态(调试用;仅管理员) |
|
||||||
| `/test <链接>` | 调试:只解析链接并返回解析结果(站点、标题、作者、标签、媒体列表),不发送任何媒体 |
|
| `/test <链接>` | 解析链接并发送媒体;不转发到频道、不弹转发前编辑提示(仅发送) |
|
||||||
|
| `/debug <链接>` | 调试:只解析链接并返回解析结果(站点、标题、作者、标签、媒体列表),不发送任何媒体 |
|
||||||
|
|
||||||
链接处理仅限私聊;命令在任意聊天可用。
|
链接处理仅限私聊;命令在任意聊天可用。在群聊里发受支持的链接会回复一条提示(改用私聊或内联查询),频道内保持静默。
|
||||||
|
|
||||||
## 备注
|
## 备注
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "x-media"
|
name = "x-media"
|
||||||
version = "1.5.0"
|
version = "1.8.0"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
@@ -11,10 +11,10 @@ regex = "1.12"
|
|||||||
html-escape = "0.2"
|
html-escape = "0.2"
|
||||||
url = "2.5.2"
|
url = "2.5.2"
|
||||||
bytes = "1"
|
bytes = "1"
|
||||||
zip = "2"
|
zip = "8"
|
||||||
tempfile = "3"
|
tempfile = "3"
|
||||||
thiserror = "2"
|
thiserror = "2"
|
||||||
rand = "0.8"
|
rand = "0.10"
|
||||||
log = "0.4"
|
log = "0.4"
|
||||||
tokio = { version = "1.40", features = ["time"] }
|
tokio = { version = "1.40", features = ["time"] }
|
||||||
|
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,6 @@
|
|||||||
|
mod interface;
|
||||||
|
mod model;
|
||||||
|
|
||||||
|
pub use interface::{
|
||||||
|
BilibiliSite, PATTERN, cache_key, enabled, fetch_from_url, is_retryable, media_headers,
|
||||||
|
};
|
||||||
@@ -0,0 +1,158 @@
|
|||||||
|
//! Serde DTOs for the Bilibili dynamic detail endpoint
|
||||||
|
//! (`/x/polymer/web-dynamic/v1/detail`), mirroring live responses
|
||||||
|
//! (field paths verified 2026-09-17). Every field is optional so an API
|
||||||
|
//! shape change degrades to "no media" instead of a parse failure.
|
||||||
|
|
||||||
|
use serde::Deserialize;
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
pub(crate) struct Detail {
|
||||||
|
/// Business code: `0` = OK, `-352`/`-412` = risk control, `500`/`4101147`
|
||||||
|
/// = gone.
|
||||||
|
pub(crate) code: i64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub(crate) message: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub(crate) data: Option<Data>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
pub(crate) struct Data {
|
||||||
|
#[serde(default)]
|
||||||
|
pub(crate) item: Option<Box<Item>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
pub(crate) struct Item {
|
||||||
|
/// The dynamic id, same numeric id as in the URL.
|
||||||
|
#[serde(default)]
|
||||||
|
pub(crate) id_str: String,
|
||||||
|
#[serde(default)]
|
||||||
|
pub(crate) modules: Option<Modules>,
|
||||||
|
/// The quoted dynamic when this item is a forward. A forward shell often
|
||||||
|
/// carries no media of its own — the original holds it.
|
||||||
|
#[serde(default)]
|
||||||
|
pub(crate) orig: Option<Box<Item>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
pub(crate) struct Modules {
|
||||||
|
#[serde(default)]
|
||||||
|
pub(crate) module_author: Option<Author>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub(crate) module_dynamic: Option<Dynamic>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
pub(crate) struct Author {
|
||||||
|
#[serde(default)]
|
||||||
|
pub(crate) name: String,
|
||||||
|
#[serde(default)]
|
||||||
|
pub(crate) mid: Option<i64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
pub(crate) struct Dynamic {
|
||||||
|
#[serde(default)]
|
||||||
|
pub(crate) desc: Option<Desc>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub(crate) major: Option<Major>,
|
||||||
|
/// A single topic (`{"id":…,"name":…}`), the dynamic's only tag source.
|
||||||
|
#[serde(default)]
|
||||||
|
pub(crate) topic: Option<Topic>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
pub(crate) struct Desc {
|
||||||
|
#[serde(default)]
|
||||||
|
pub(crate) text: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `major` is a tagged union: `type` (`MAJOR_TYPE_DRAW` / `_OPUS` /
|
||||||
|
/// `_ARCHIVE` / …) plus one payload object per type. Only the three payloads
|
||||||
|
/// this adapter reads are modeled; an unknown major simply yields no media.
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
pub(crate) struct Major {
|
||||||
|
#[serde(default)]
|
||||||
|
pub(crate) draw: Option<Draw>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub(crate) opus: Option<Opus>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub(crate) archive: Option<Archive>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
pub(crate) struct Draw {
|
||||||
|
#[serde(default)]
|
||||||
|
pub(crate) items: Vec<Pic>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
pub(crate) struct Pic {
|
||||||
|
/// `major.draw` image URL.
|
||||||
|
#[serde(default)]
|
||||||
|
pub(crate) src: Option<String>,
|
||||||
|
/// `major.opus.pics` image URL — the opus shape names the field
|
||||||
|
/// differently while carrying the same image.
|
||||||
|
#[serde(default)]
|
||||||
|
pub(crate) url: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Pic {
|
||||||
|
/// The image URL, whichever key this serialization put it under.
|
||||||
|
pub(crate) fn url(&self) -> Option<&str> {
|
||||||
|
self.src.as_deref().or(self.url.as_deref())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `major.opus`: the serialization of an image/text post the web client asks
|
||||||
|
/// for (`features=itemOpusStyle`). It carries the parts the legacy shape drops
|
||||||
|
/// entirely — the document title and body of an opus post, whose
|
||||||
|
/// `module_dynamic.desc` comes back `null`.
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
pub(crate) struct Opus {
|
||||||
|
/// Document headline; often absent.
|
||||||
|
#[serde(default)]
|
||||||
|
pub(crate) title: Option<String>,
|
||||||
|
/// Document body (untruncated: a 307-char sample came back whole).
|
||||||
|
#[serde(default)]
|
||||||
|
pub(crate) summary: Option<Desc>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub(crate) pics: Vec<Pic>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
pub(crate) struct Archive {
|
||||||
|
/// The attached video's cover — the only image an AV dynamic has (the
|
||||||
|
/// video itself is deliberately not resolved, see the module docs).
|
||||||
|
#[serde(default)]
|
||||||
|
pub(crate) cover: Option<String>,
|
||||||
|
/// The video's title. An AV dynamic has no body of its own (`desc` comes
|
||||||
|
/// back `null`), so this card title is the post's content.
|
||||||
|
#[serde(default)]
|
||||||
|
pub(crate) title: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
pub(crate) struct Topic {
|
||||||
|
#[serde(default)]
|
||||||
|
pub(crate) name: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Response of the anonymous fingerprint endpoint (`/x/frontend/finger/spi`),
|
||||||
|
/// the source of the adapter's device cookies.
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
pub(crate) struct Fingerprint {
|
||||||
|
#[serde(default)]
|
||||||
|
pub(crate) data: Option<FingerprintData>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
pub(crate) struct FingerprintData {
|
||||||
|
/// Sent as the `buvid3` cookie.
|
||||||
|
#[serde(default, rename = "b_3")]
|
||||||
|
pub(crate) buvid3: String,
|
||||||
|
/// Sent as the `buvid4` cookie.
|
||||||
|
#[serde(default, rename = "b_4")]
|
||||||
|
pub(crate) buvid4: String,
|
||||||
|
}
|
||||||
@@ -330,13 +330,16 @@ impl From<Post> for Fetched {
|
|||||||
url: url.clone(),
|
url: url.clone(),
|
||||||
author: encode_text(&post.author).into_owned(),
|
author: encode_text(&post.author).into_owned(),
|
||||||
author_url: author_url.clone(),
|
author_url: author_url.clone(),
|
||||||
title: encode_text(&post.text).into_owned(),
|
// A post has no title: its text is all content.
|
||||||
|
title: String::new(),
|
||||||
|
content: encode_text(&post.text).into_owned(),
|
||||||
tags: String::new(),
|
tags: String::new(),
|
||||||
});
|
});
|
||||||
Fetched {
|
Fetched {
|
||||||
source_url: url,
|
source_url: url,
|
||||||
caption: post.caption(),
|
caption: post.caption(),
|
||||||
title: post.text.clone(),
|
title: String::new(),
|
||||||
|
content: post.text.clone(),
|
||||||
media: post.media,
|
media: post.media,
|
||||||
sensitive: post.sensitive,
|
sensitive: post.sensitive,
|
||||||
site_id: "bsky",
|
site_id: "bsky",
|
||||||
@@ -410,7 +413,8 @@ mod tests {
|
|||||||
fetched.source_url,
|
fetched.source_url,
|
||||||
"https://bsky.app/profile/user.bsky.social/post/3xxxx"
|
"https://bsky.app/profile/user.bsky.social/post/3xxxx"
|
||||||
);
|
);
|
||||||
assert_eq!(fetched.title, "hello <world>");
|
assert_eq!(fetched.title, "");
|
||||||
|
assert_eq!(fetched.content, "hello <world>");
|
||||||
assert_eq!(fetched.media.len(), 1);
|
assert_eq!(fetched.media.len(), 1);
|
||||||
assert!(!fetched.sensitive);
|
assert!(!fetched.sensitive);
|
||||||
// display_name absent -> empty fallback
|
// display_name absent -> empty fallback
|
||||||
|
|||||||
@@ -123,21 +123,23 @@ impl From<model::Note> for Fetched {
|
|||||||
let cw = content.cw.as_deref().unwrap_or_default();
|
let cw = content.cw.as_deref().unwrap_or_default();
|
||||||
// Notes carry hashtags inline in the text (no structured tags array);
|
// Notes carry hashtags inline in the text (no structured tags array);
|
||||||
// a CW note gets the marker prefixed so recipients see the spoiler.
|
// a CW note gets the marker prefixed so recipients see the spoiler.
|
||||||
let mut title = cw.to_string();
|
let mut text = cw.to_string();
|
||||||
if !cw.is_empty() && !title.ends_with(' ') {
|
if !cw.is_empty() && !text.ends_with(' ') {
|
||||||
title.push(' ');
|
text.push(' ');
|
||||||
}
|
}
|
||||||
title.push_str(content.text.as_deref().unwrap_or_default().trim());
|
text.push_str(content.text.as_deref().unwrap_or_default().trim());
|
||||||
let title = title.trim().to_string();
|
let text = text.trim().to_string();
|
||||||
|
|
||||||
let caption = caption(&url, &author_url, &author, &title);
|
let caption = caption(&url, &author_url, &author, &text);
|
||||||
let sensitive = content.cw.is_some() || content.files.iter().any(|f| f.is_sensitive);
|
let sensitive = content.cw.is_some() || content.files.iter().any(|f| f.is_sensitive);
|
||||||
let media: Vec<Media> = content.files.iter().filter_map(media_from_file).collect();
|
let media: Vec<Media> = content.files.iter().filter_map(media_from_file).collect();
|
||||||
|
|
||||||
Fetched {
|
Fetched {
|
||||||
source_url: url.clone(),
|
source_url: url.clone(),
|
||||||
caption,
|
caption,
|
||||||
title: title.clone(),
|
// A note has no title: its text (CW marker included) is content.
|
||||||
|
title: String::new(),
|
||||||
|
content: text.clone(),
|
||||||
media,
|
media,
|
||||||
sensitive,
|
sensitive,
|
||||||
site_id: "misskey",
|
site_id: "misskey",
|
||||||
@@ -145,7 +147,8 @@ impl From<model::Note> for Fetched {
|
|||||||
url,
|
url,
|
||||||
author: encode_text(&author).into_owned(),
|
author: encode_text(&author).into_owned(),
|
||||||
author_url: author_url.clone(),
|
author_url: author_url.clone(),
|
||||||
title: encode_text(&title).into_owned(),
|
title: String::new(),
|
||||||
|
content: encode_text(&text).into_owned(),
|
||||||
tags: String::new(),
|
tags: String::new(),
|
||||||
}),
|
}),
|
||||||
_keep_alive: None,
|
_keep_alive: None,
|
||||||
@@ -257,7 +260,8 @@ mod tests {
|
|||||||
"https://misskey.io/notes/aotihl10lqrs015s"
|
"https://misskey.io/notes/aotihl10lqrs015s"
|
||||||
);
|
);
|
||||||
assert_eq!(fetched.site_id, "misskey");
|
assert_eq!(fetched.site_id, "misskey");
|
||||||
assert_eq!(fetched.title, "hello");
|
assert_eq!(fetched.title, "");
|
||||||
|
assert_eq!(fetched.content, "hello");
|
||||||
assert!(fetched.sensitive);
|
assert!(fetched.sensitive);
|
||||||
assert_eq!(fetched.media.len(), 1);
|
assert_eq!(fetched.media.len(), 1);
|
||||||
match &fetched.media[0] {
|
match &fetched.media[0] {
|
||||||
@@ -308,7 +312,8 @@ mod tests {
|
|||||||
note["text"] = serde_json::json!("body");
|
note["text"] = serde_json::json!("body");
|
||||||
let fetched: Fetched = note_json(note).into();
|
let fetched: Fetched = note_json(note).into();
|
||||||
assert!(fetched.sensitive);
|
assert!(fetched.sensitive);
|
||||||
assert_eq!(fetched.title, "spoiler body");
|
assert_eq!(fetched.title, "");
|
||||||
|
assert_eq!(fetched.content, "spoiler body");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -341,7 +346,8 @@ mod tests {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
let fetched: Fetched = note_json(note).into();
|
let fetched: Fetched = note_json(note).into();
|
||||||
assert_eq!(fetched.title, "inner text");
|
assert_eq!(fetched.title, "");
|
||||||
|
assert_eq!(fetched.content, "inner text");
|
||||||
assert_eq!(fetched.media.len(), 1);
|
assert_eq!(fetched.media.len(), 1);
|
||||||
// The source URL still points at the renote shell the user posted.
|
// The source URL still points at the renote shell the user posted.
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
|
|||||||
+183
-32
@@ -1,8 +1,8 @@
|
|||||||
//! Site fetching dispatcher and unified result types.
|
//! Site fetching dispatcher and unified result types.
|
||||||
//!
|
//!
|
||||||
//! Dispatch order: twitter → bsky → misskey → pixiv. Each site module
|
//! Dispatch order: twitter → bsky → misskey → pixiv → bilibili. Each site
|
||||||
//! exports a `PATTERN`, `enabled()` and `fetch_from_url()`; a future site
|
//! module exports a `PATTERN`, `enabled()` and `fetch_from_url()`; a future
|
||||||
//! plugs in by adding one guarded entry in [`fetch_once`].
|
//! site plugs in by adding one guarded entry in `SITES`.
|
||||||
|
|
||||||
use std::future::Future;
|
use std::future::Future;
|
||||||
use std::pin::Pin;
|
use std::pin::Pin;
|
||||||
@@ -13,6 +13,7 @@ use std::time::Duration;
|
|||||||
use regex::Regex;
|
use regex::Regex;
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
|
|
||||||
|
pub mod bilibili;
|
||||||
pub mod bsky;
|
pub mod bsky;
|
||||||
pub mod misskey;
|
pub mod misskey;
|
||||||
pub mod pixiv;
|
pub mod pixiv;
|
||||||
@@ -20,22 +21,31 @@ pub mod twitter;
|
|||||||
|
|
||||||
pub use pixiv::PixivError;
|
pub use pixiv::PixivError;
|
||||||
|
|
||||||
/// The result of fetching a post: canonical URL, HTML caption, raw text,
|
/// The result of fetching a post: canonical URL, HTML caption, the post's
|
||||||
/// media list and spoiler flag. Produced by [`fetch`].
|
/// title and body, media list and spoiler flag. Produced by [`fetch`].
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct Fetched {
|
pub struct Fetched {
|
||||||
/// Canonical URL: x.com/{author}/status/{id} |
|
/// Canonical URL: `x.com/{author}/status/{id}` |
|
||||||
/// https://www.pixiv.net/artworks/{id} |
|
/// `https://www.pixiv.net/artworks/{id}` |
|
||||||
/// https://bsky.app/profile/{handle}/post/{rkey}
|
/// `https://bsky.app/profile/{handle}/post/{rkey}` |
|
||||||
|
/// `https://www.bilibili.com/opus/{id}`
|
||||||
pub source_url: String,
|
pub source_url: String,
|
||||||
/// The exact HTML produced by the site's caption().
|
/// The exact HTML produced by the site's caption().
|
||||||
pub caption: String,
|
pub caption: String,
|
||||||
/// Raw post text (tweet text / bsky text / pixiv title).
|
/// The post's own title, where the platform has one: a pixiv artwork's
|
||||||
|
/// title, the headline of a bilibili opus post or the title of the video
|
||||||
|
/// an AV dynamic attaches. Empty on the platforms whose posts are text
|
||||||
|
/// only (x/twitter, bsky, misskey) and on bilibili posts without a
|
||||||
|
/// headline.
|
||||||
pub title: String,
|
pub title: String,
|
||||||
|
/// The post's body text, as the platform exposes it: a tweet, a bsky or
|
||||||
|
/// misskey post, a bilibili dynamic's text, a pixiv artwork's description
|
||||||
|
/// (HTML flattened). Empty when the post has no text at all.
|
||||||
|
pub content: String,
|
||||||
pub media: Vec<crate::media::Media>,
|
pub media: Vec<crate::media::Media>,
|
||||||
/// Spoiler flag for all media of this post.
|
/// Spoiler flag for all media of this post.
|
||||||
pub sensitive: bool,
|
pub sensitive: bool,
|
||||||
/// Site id (`"twitter"` / `"bsky"` / `"pixiv"`): the single source of
|
/// Site id (`"twitter"` / `"bsky"` / `"pixiv"` / `"bilibili"`): the single source of
|
||||||
/// truth for site identity — caption-format lookup, cache-key prefix and
|
/// truth for site identity — caption-format lookup, cache-key prefix and
|
||||||
/// the SetFormat whitelist all derive from it. Set by the producing site.
|
/// the SetFormat whitelist all derive from it. Set by the producing site.
|
||||||
pub site_id: &'static str,
|
pub site_id: &'static str,
|
||||||
@@ -46,17 +56,38 @@ pub struct Fetched {
|
|||||||
pub(crate) _keep_alive: Option<tempfile::TempDir>,
|
pub(crate) _keep_alive: Option<tempfile::TempDir>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Pre-escaped values for `{url} {author} {author_url} {title} {tags}`
|
/// Values for the `{url} {author} {author_url} {title} {content} {tags}`
|
||||||
/// placeholders in user-supplied caption formats.
|
/// placeholders in user-supplied caption formats, substituted by
|
||||||
|
/// [`caption_from_fields`] as HTML text (never as an attribute value).
|
||||||
|
///
|
||||||
|
/// `author`, `title`, `content` and `tags` come from the site API (post
|
||||||
|
/// text, display names, descriptions) and are HTML-escaped at construction.
|
||||||
|
/// `url` and `author_url` stay raw: they are canonical URLs the adapter
|
||||||
|
/// builds from numeric ids and API-constrained handles/DIDs, so they carry
|
||||||
|
/// no escapable character — the bot's `/test` report relies on that when it
|
||||||
|
/// embeds them.
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub(crate) struct RenderData {
|
pub(crate) struct RenderData {
|
||||||
pub url: String,
|
pub url: String,
|
||||||
pub author: String,
|
pub author: String,
|
||||||
pub author_url: String,
|
pub author_url: String,
|
||||||
pub title: String,
|
pub title: String,
|
||||||
|
pub content: String,
|
||||||
pub tags: String,
|
pub tags: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The post's text as one string: title and content joined by a line break,
|
||||||
|
/// each only when it is non-empty. This is what the sites' built-in captions
|
||||||
|
/// show after the author line, and what the bot quotes when it is long.
|
||||||
|
pub fn compose_text(title: &str, content: &str) -> String {
|
||||||
|
match (title.is_empty(), content.is_empty()) {
|
||||||
|
(false, false) => format!("{title}\n{content}"),
|
||||||
|
(false, true) => title.to_string(),
|
||||||
|
(true, false) => content.to_string(),
|
||||||
|
(true, true) => String::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl Fetched {
|
impl Fetched {
|
||||||
/// The site this post came from (used for per-site format overrides).
|
/// The site this post came from (used for per-site format overrides).
|
||||||
/// A thin alias over [`Fetched::site_id`] kept for callers that read the
|
/// A thin alias over [`Fetched::site_id`] kept for callers that read the
|
||||||
@@ -80,21 +111,23 @@ impl Fetched {
|
|||||||
&data.author,
|
&data.author,
|
||||||
&data.author_url,
|
&data.author_url,
|
||||||
&data.title,
|
&data.title,
|
||||||
|
&data.content,
|
||||||
&data.tags,
|
&data.tags,
|
||||||
),
|
),
|
||||||
_ => truncate_caption(&self.caption),
|
_ => truncate_caption(&self.caption),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The pre-escaped placeholder values (author, author_url, title, tags)
|
/// The pre-escaped placeholder values (author, author_url, title,
|
||||||
/// a caller needs to rebuild a caption later, e.g. for a cached post
|
/// content, tags) a caller needs to rebuild a caption later, e.g. for a
|
||||||
/// where the [`Fetched`] is no longer available.
|
/// cached post where the [`Fetched`] is no longer available.
|
||||||
pub fn render_fields(&self) -> Option<(&str, &str, &str, &str)> {
|
pub fn render_fields(&self) -> Option<(&str, &str, &str, &str, &str)> {
|
||||||
self.render_data.as_ref().map(|d| {
|
self.render_data.as_ref().map(|d| {
|
||||||
(
|
(
|
||||||
d.author.as_str(),
|
d.author.as_str(),
|
||||||
d.author_url.as_str(),
|
d.author_url.as_str(),
|
||||||
d.title.as_str(),
|
d.title.as_str(),
|
||||||
|
d.content.as_str(),
|
||||||
d.tags.as_str(),
|
d.tags.as_str(),
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
@@ -140,6 +173,11 @@ pub fn truncate_caption(caption: &str) -> String {
|
|||||||
/// [`Fetched::caption_with`]. An empty format returns `built_in` unchanged.
|
/// [`Fetched::caption_with`]. An empty format returns `built_in` unchanged.
|
||||||
/// The result is truncated to [`MAX_CAPTION_CHARS`] (Telegram's caption
|
/// The result is truncated to [`MAX_CAPTION_CHARS`] (Telegram's caption
|
||||||
/// limit for HTML parse mode).
|
/// limit for HTML parse mode).
|
||||||
|
///
|
||||||
|
/// One flat argument per placeholder keeps the two callers (the fresh and the
|
||||||
|
/// cached caption path) mirroring each other; the same shape as the bot's
|
||||||
|
/// `debug_report`.
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
pub fn caption_from_fields(
|
pub fn caption_from_fields(
|
||||||
format: &str,
|
format: &str,
|
||||||
built_in: &str,
|
built_in: &str,
|
||||||
@@ -147,6 +185,7 @@ pub fn caption_from_fields(
|
|||||||
author: &str,
|
author: &str,
|
||||||
author_url: &str,
|
author_url: &str,
|
||||||
title: &str,
|
title: &str,
|
||||||
|
content: &str,
|
||||||
tags: &str,
|
tags: &str,
|
||||||
) -> String {
|
) -> String {
|
||||||
if format.is_empty() {
|
if format.is_empty() {
|
||||||
@@ -159,6 +198,7 @@ pub fn caption_from_fields(
|
|||||||
.replace("{author}", author)
|
.replace("{author}", author)
|
||||||
.replace("{author_url}", author_url)
|
.replace("{author_url}", author_url)
|
||||||
.replace("{title}", title)
|
.replace("{title}", title)
|
||||||
|
.replace("{content}", content)
|
||||||
.replace("{tags}", tags),
|
.replace("{tags}", tags),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -166,7 +206,7 @@ pub fn caption_from_fields(
|
|||||||
/// Stable per-post cache key derived from any supported URL, so variant
|
/// Stable per-post cache key derived from any supported URL, so variant
|
||||||
/// domains (x.com / twitter.com / fxtwitter.com, mobile, `/photo/N`
|
/// domains (x.com / twitter.com / fxtwitter.com, mobile, `/photo/N`
|
||||||
/// suffixes) map to the same post. Delegates to each registered site's
|
/// suffixes) map to the same post. Delegates to each registered site's
|
||||||
/// `cache_key` (dispatch order twitter → bsky → pixiv).
|
/// `cache_key` (in registry order).
|
||||||
pub fn cache_key(url: &str) -> Option<String> {
|
pub fn cache_key(url: &str) -> Option<String> {
|
||||||
SITES.iter().find_map(|site| site.cache_key(url))
|
SITES.iter().find_map(|site| site.cache_key(url))
|
||||||
}
|
}
|
||||||
@@ -207,6 +247,12 @@ pub enum FetchError {
|
|||||||
NotFound,
|
NotFound,
|
||||||
#[error("blocked")]
|
#[error("blocked")]
|
||||||
Blocked,
|
Blocked,
|
||||||
|
/// The URL matches a registered site that is disabled right now (pixiv
|
||||||
|
/// without `PIXIV_REFRESH_TOKEN`, or after a failed login). Distinct from
|
||||||
|
/// `Ok(None)` — an unsupported link — so the bot can tell the user why
|
||||||
|
/// the link was not handled instead of silently ignoring it.
|
||||||
|
#[error("{site} support is disabled")]
|
||||||
|
Disabled { site: &'static str },
|
||||||
/// The post exists but its content is withheld (twitter NSFW /
|
/// The post exists but its content is withheld (twitter NSFW /
|
||||||
/// age-restricted tweets come back as an empty `{}` from syndication).
|
/// age-restricted tweets come back as an empty `{}` from syndication).
|
||||||
#[error("content withheld (sensitive)")]
|
#[error("content withheld (sensitive)")]
|
||||||
@@ -275,20 +321,21 @@ pub(crate) fn log_once_ffmpeg_missing() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Site adapter: one impl per supported site (twitter / bsky / pixiv),
|
/// Site adapter: one impl per supported site (twitter / bsky / misskey /
|
||||||
/// registered in [`SITES`]. All site-specific knowledge — URL pattern,
|
/// pixiv / bilibili), registered in `SITES`. All site-specific knowledge — URL pattern,
|
||||||
/// cache-key format, fetch, retry policy, media-host headers, startup
|
/// cache-key format, fetch, retry policy, media-host headers, startup
|
||||||
/// validation — lives in the site module; the central dispatcher only
|
/// validation — lives in the site module; the central dispatcher only
|
||||||
/// iterates the registry.
|
/// iterates the registry.
|
||||||
///
|
///
|
||||||
/// Async methods return a boxed future (see [`SiteFuture`]): `async fn` /
|
/// Async methods return a boxed future (see `SiteFuture`): `async fn` /
|
||||||
/// RPITIT in traits are not dyn-compatible (verified on rustc 1.95), and
|
/// RPITIT in traits are not dyn-compatible (verified on rustc 1.95), and
|
||||||
/// `+ Send` is required since URL/queue workers spawn these futures. The
|
/// `+ Send` is required since URL/queue workers spawn these futures. The
|
||||||
/// site structs are stateless unit structs, so the boxed futures never
|
/// site structs are stateless unit structs, so the boxed futures never
|
||||||
/// borrow from `self` beyond the call's scope.
|
/// borrow from `self` beyond the call's scope.
|
||||||
pub trait Site: Send + Sync {
|
pub trait Site: Send + Sync {
|
||||||
/// Stable site id (`"twitter"` / `"bsky"` / `"pixiv"`): caption-format
|
/// Stable site id (`"twitter"` / `"bsky"` / `"misskey"` / `"pixiv"` /
|
||||||
/// lookup, cache-key prefixes and the SetFormat whitelist derive from it.
|
/// `"bilibili"`): caption-format lookup, cache-key prefixes and the
|
||||||
|
/// SetFormat whitelist derive from it.
|
||||||
fn id(&self) -> &'static str;
|
fn id(&self) -> &'static str;
|
||||||
/// URL pattern; the dispatcher's first match wins (dispatch order).
|
/// URL pattern; the dispatcher's first match wins (dispatch order).
|
||||||
fn pattern(&self) -> &'static Regex;
|
fn pattern(&self) -> &'static Regex;
|
||||||
@@ -324,14 +371,15 @@ pub trait Site: Send + Sync {
|
|||||||
type SiteFuture<'a, T, E = FetchError> = Pin<Box<dyn Future<Output = Result<T, E>> + Send + 'a>>;
|
type SiteFuture<'a, T, E = FetchError> = Pin<Box<dyn Future<Output = Result<T, E>> + Send + 'a>>;
|
||||||
|
|
||||||
/// The one registry of supported sites, in dispatch order (twitter → bsky →
|
/// The one registry of supported sites, in dispatch order (twitter → bsky →
|
||||||
/// pixiv). Adding a site = new module + one `Box::new(...)` entry here; the
|
/// misskey → pixiv → bilibili). Adding a site = new module + one
|
||||||
/// bot crate never lists sites itself.
|
/// `Box::new(...)` entry here; the bot crate never lists sites itself.
|
||||||
static SITES: LazyLock<Vec<Box<dyn Site>>> = LazyLock::new(|| {
|
static SITES: LazyLock<Vec<Box<dyn Site>>> = LazyLock::new(|| {
|
||||||
vec![
|
vec![
|
||||||
Box::new(twitter::TwitterSite),
|
Box::new(twitter::TwitterSite),
|
||||||
Box::new(bsky::BskySite),
|
Box::new(bsky::BskySite),
|
||||||
Box::new(misskey::MisskeySite),
|
Box::new(misskey::MisskeySite),
|
||||||
Box::new(pixiv::PixivSite),
|
Box::new(pixiv::PixivSite),
|
||||||
|
Box::new(bilibili::BilibiliSite),
|
||||||
]
|
]
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -343,6 +391,17 @@ fn find_site(url: &str) -> Option<&'static dyn Site> {
|
|||||||
.map(|site| site.as_ref())
|
.map(|site| site.as_ref())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The site whose pattern matches `url` but which is disabled right now.
|
||||||
|
/// `None` when no site matches the URL at all, or when the matching site is
|
||||||
|
/// enabled. Lets the dispatcher tell "unsupported link" (silently ignored)
|
||||||
|
/// apart from "this bot has that site switched off" (reported to the user).
|
||||||
|
fn disabled_site(url: &str) -> Option<&'static str> {
|
||||||
|
SITES
|
||||||
|
.iter()
|
||||||
|
.find(|site| !site.enabled() && site.pattern().is_match(url))
|
||||||
|
.map(|site| site.id())
|
||||||
|
}
|
||||||
|
|
||||||
/// Every supported site id, in dispatch order. The bot's SetFormat whitelist
|
/// Every supported site id, in dispatch order. The bot's SetFormat whitelist
|
||||||
/// derives from this list.
|
/// derives from this list.
|
||||||
pub fn site_ids() -> Vec<&'static str> {
|
pub fn site_ids() -> Vec<&'static str> {
|
||||||
@@ -366,7 +425,9 @@ pub async fn validate_all() -> Vec<(&'static str, String)> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Fetches a post from its URL. Returns `Ok(None)` when no site pattern
|
/// Fetches a post from its URL. Returns `Ok(None)` when no site pattern
|
||||||
/// matches (unsupported links are silently ignored by the bot).
|
/// matches (unsupported links are silently ignored by the bot) and
|
||||||
|
/// [`FetchError::Disabled`] when the URL belongs to a registered site that is
|
||||||
|
/// switched off right now — the two are different answers for the user.
|
||||||
///
|
///
|
||||||
/// Transient failures are retried: 3 total attempts with 1s then 2s delays.
|
/// Transient failures are retried: 3 total attempts with 1s then 2s delays.
|
||||||
/// What counts as transient is the matched site's own policy (`is_retryable`
|
/// What counts as transient is the matched site's own policy (`is_retryable`
|
||||||
@@ -375,10 +436,30 @@ pub async fn validate_all() -> Vec<(&'static str, String)> {
|
|||||||
/// are returned immediately; retrying them only wastes attempts against the
|
/// are returned immediately; retrying them only wastes attempts against the
|
||||||
/// source site.
|
/// source site.
|
||||||
pub async fn fetch(url: &str) -> Result<Option<Fetched>, FetchError> {
|
pub async fn fetch(url: &str) -> Result<Option<Fetched>, FetchError> {
|
||||||
|
fetch_with_attempts(url, MAX_FETCH_ATTEMPTS).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// [`fetch`] without the retry backoff (one attempt). For callers with a
|
||||||
|
/// short deadline: an inline query's answer window is measured in seconds, so
|
||||||
|
/// the 1s + 2s retry sleeps would outlast the query the answer belongs to.
|
||||||
|
pub async fn fetch_once(url: &str) -> Result<Option<Fetched>, FetchError> {
|
||||||
|
fetch_with_attempts(url, 1).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Total attempts of the retried [`fetch`] (3: the initial try plus two).
|
||||||
|
const MAX_FETCH_ATTEMPTS: u32 = 3;
|
||||||
|
|
||||||
|
async fn fetch_with_attempts(url: &str, attempts: u32) -> Result<Option<Fetched>, FetchError> {
|
||||||
let Some(site) = find_site(url) else {
|
let Some(site) = find_site(url) else {
|
||||||
return Ok(None);
|
// A registered-but-disabled site (pixiv without a token) is not an
|
||||||
|
// unsupported link: report it, so the bot answers the user instead of
|
||||||
|
// ignoring the message.
|
||||||
|
return match disabled_site(url) {
|
||||||
|
Some(site) => Err(FetchError::Disabled { site }),
|
||||||
|
None => Ok(None),
|
||||||
|
};
|
||||||
};
|
};
|
||||||
for attempt in 0..3u32 {
|
for attempt in 0..attempts.max(1) {
|
||||||
match site.fetch_from_url(url).await {
|
match site.fetch_from_url(url).await {
|
||||||
Ok(fetched) => {
|
Ok(fetched) => {
|
||||||
// Per-request detail: debug only, keyed by the post id.
|
// Per-request detail: debug only, keyed by the post id.
|
||||||
@@ -391,7 +472,7 @@ pub async fn fetch(url: &str) -> Result<Option<Fetched>, FetchError> {
|
|||||||
return Ok(Some(fetched));
|
return Ok(Some(fetched));
|
||||||
}
|
}
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
if site.is_retryable(&err) && attempt < 2 {
|
if site.is_retryable(&err) && attempt + 1 < attempts {
|
||||||
tokio::time::sleep(Duration::from_secs(1 << attempt)).await;
|
tokio::time::sleep(Duration::from_secs(1 << attempt)).await;
|
||||||
} else {
|
} else {
|
||||||
return Err(err);
|
return Err(err);
|
||||||
@@ -402,6 +483,15 @@ pub async fn fetch(url: &str) -> Result<Option<Fetched>, FetchError> {
|
|||||||
unreachable!("retry loop always returns")
|
unreachable!("retry loop always returns")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Whether fetching `url` requires site-specific headers (pixiv's `Referer`
|
||||||
|
/// for `pximg.net` hotlink protection, see [`Site::media_headers`]). Telegram's
|
||||||
|
/// own fetch of a media URL sends none of them, so a URL that needs them fails
|
||||||
|
/// there — callers that hand a URL to Telegram (inline query results) must
|
||||||
|
/// skip such media instead of shipping a broken item.
|
||||||
|
pub fn needs_media_headers(url: &str) -> bool {
|
||||||
|
SITES.iter().any(|site| site.media_headers(url).is_some())
|
||||||
|
}
|
||||||
|
|
||||||
/// Applies every site's media-header rule to a download request (pixiv's
|
/// Applies every site's media-header rule to a download request (pixiv's
|
||||||
/// `Referer` for pximg.net hotlink protection). Sites contribute via their
|
/// `Referer` for pximg.net hotlink protection). Sites contribute via their
|
||||||
/// `media_headers(url)` — the central download code carries no per-site logic.
|
/// `media_headers(url)` — the central download code carries no per-site logic.
|
||||||
@@ -519,6 +609,10 @@ mod tests {
|
|||||||
cache_key("https://bsky.app/profile/handle.example/post/3lorem"),
|
cache_key("https://bsky.app/profile/handle.example/post/3lorem"),
|
||||||
Some("bsky:handle.example/3lorem".into())
|
Some("bsky:handle.example/3lorem".into())
|
||||||
);
|
);
|
||||||
|
assert_eq!(
|
||||||
|
cache_key("https://t.bilibili.com/1245284537985925159"),
|
||||||
|
Some("bilibili:1245284537985925159".into())
|
||||||
|
);
|
||||||
assert_eq!(cache_key("https://example.com/not-a-post"), None);
|
assert_eq!(cache_key("https://example.com/not-a-post"), None);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -527,16 +621,21 @@ mod tests {
|
|||||||
assert_eq!(site_id_from_key("twitter:123"), "twitter");
|
assert_eq!(site_id_from_key("twitter:123"), "twitter");
|
||||||
assert_eq!(site_id_from_key("pixiv:123"), "pixiv");
|
assert_eq!(site_id_from_key("pixiv:123"), "pixiv");
|
||||||
assert_eq!(site_id_from_key("bsky:handle.example/3lorem"), "bsky");
|
assert_eq!(site_id_from_key("bsky:handle.example/3lorem"), "bsky");
|
||||||
|
assert_eq!(site_id_from_key("bilibili:123"), "bilibili");
|
||||||
assert_eq!(site_id_from_key("unknown:1"), "unknown");
|
assert_eq!(site_id_from_key("unknown:1"), "unknown");
|
||||||
assert_eq!(site_id_from_key("no-colon"), "unknown");
|
assert_eq!(site_id_from_key("no-colon"), "unknown");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn registry_lists_all_sites_in_dispatch_order() {
|
fn registry_lists_all_sites_in_dispatch_order() {
|
||||||
assert_eq!(site_ids(), vec!["twitter", "bsky", "misskey", "pixiv"]);
|
assert_eq!(
|
||||||
|
site_ids(),
|
||||||
|
vec!["twitter", "bsky", "misskey", "pixiv", "bilibili"]
|
||||||
|
);
|
||||||
// Enabled sites dispatch; unsupported URLs never match.
|
// Enabled sites dispatch; unsupported URLs never match.
|
||||||
assert!(find_site("https://x.com/u/status/1").is_some());
|
assert!(find_site("https://x.com/u/status/1").is_some());
|
||||||
assert!(find_site("https://misskey.io/notes/abc").is_some());
|
assert!(find_site("https://misskey.io/notes/abc").is_some());
|
||||||
|
assert!(find_site("https://t.bilibili.com/1245284537985925159").is_some());
|
||||||
assert!(find_site("https://example.com/x").is_none());
|
assert!(find_site("https://example.com/x").is_none());
|
||||||
// Cache keys are pattern-driven, independent of the enabled() gate
|
// Cache keys are pattern-driven, independent of the enabled() gate
|
||||||
// (pixiv is disabled in tests without PIXIV_REFRESH_TOKEN).
|
// (pixiv is disabled in tests without PIXIV_REFRESH_TOKEN).
|
||||||
@@ -564,25 +663,34 @@ mod tests {
|
|||||||
// The format string is escaped, the field values are substituted
|
// The format string is escaped, the field values are substituted
|
||||||
// verbatim (callers pass the already-escaped render data).
|
// verbatim (callers pass the already-escaped render data).
|
||||||
let out = caption_from_fields(
|
let out = caption_from_fields(
|
||||||
"see {author} at {url} — {title}",
|
"see {author} at {url} — {title}: {content}",
|
||||||
"",
|
"",
|
||||||
"https://x.com/u/status/1",
|
"https://x.com/u/status/1",
|
||||||
"A & B",
|
"A & B",
|
||||||
"https://x.com/u",
|
"https://x.com/u",
|
||||||
"hello <world>",
|
"hello <world>",
|
||||||
|
"the body",
|
||||||
"",
|
"",
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
out,
|
out,
|
||||||
"see A & B at https://x.com/u/status/1 — hello <world>"
|
"see A & B at https://x.com/u/status/1 — hello <world>: the body"
|
||||||
);
|
);
|
||||||
// Empty format keeps the built-in caption untouched.
|
// Empty format keeps the built-in caption untouched.
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
caption_from_fields("", "built-in", "u", "a", "au", "t", "g"),
|
caption_from_fields("", "built-in", "u", "a", "au", "t", "c", "g"),
|
||||||
"built-in"
|
"built-in"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn compose_text_joins_title_and_content() {
|
||||||
|
assert_eq!(compose_text("标题", "正文"), "标题\n正文");
|
||||||
|
assert_eq!(compose_text("标题", ""), "标题");
|
||||||
|
assert_eq!(compose_text("", "正文"), "正文");
|
||||||
|
assert_eq!(compose_text("", ""), "");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn truncate_caption_keeps_short_text() {
|
fn truncate_caption_keeps_short_text() {
|
||||||
assert_eq!(truncate_caption("short"), "short");
|
assert_eq!(truncate_caption("short"), "short");
|
||||||
@@ -634,6 +742,49 @@ mod tests {
|
|||||||
assert!(matches!(result, Ok(None)), "got {result:?}");
|
assert!(matches!(result, Ok(None)), "got {result:?}");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn media_headers_are_reported_only_where_telegram_would_fail() {
|
||||||
|
// pixiv's CDN needs a Referer, which only the bot can send: an inline
|
||||||
|
// result pointing at it renders broken, so callers skip it.
|
||||||
|
assert!(needs_media_headers(
|
||||||
|
"https://i.pximg.net/img-original/img/2024/01/01/00/00/00/1_p0.jpg"
|
||||||
|
));
|
||||||
|
// The rest serve direct requests (verified per site in their modules).
|
||||||
|
for url in [
|
||||||
|
"https://pbs.twimg.com/media/1.jpg",
|
||||||
|
"https://cdn.bsky.app/img/1.jpg",
|
||||||
|
"https://media.misskeyusercontent.jp/io/1.webp",
|
||||||
|
"https://i0.hdslb.com/bfs/1.jpg",
|
||||||
|
] {
|
||||||
|
assert!(!needs_media_headers(url), "{url}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn disabled_site_is_reported_not_ignored() {
|
||||||
|
// pixiv is the only token-gated site; with PIXIV_REFRESH_TOKEN set it
|
||||||
|
// is enabled and this link would hit the network, so skip then.
|
||||||
|
if std::env::var("PIXIV_REFRESH_TOKEN")
|
||||||
|
.ok()
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.is_some()
|
||||||
|
{
|
||||||
|
eprintln!("skipping: PIXIV_REFRESH_TOKEN is set");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let result = fetch("https://www.pixiv.net/artworks/1").await;
|
||||||
|
assert!(
|
||||||
|
matches!(result, Err(FetchError::Disabled { site: "pixiv" })),
|
||||||
|
"got {result:?}"
|
||||||
|
);
|
||||||
|
// The cache key still resolves: the bot keys the reply and the link
|
||||||
|
// cache off it even when the site is off.
|
||||||
|
assert_eq!(
|
||||||
|
cache_key("https://www.pixiv.net/artworks/1"),
|
||||||
|
Some("pixiv:1".into())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn download_media_pixiv_original_with_referer() {
|
async fn download_media_pixiv_original_with_referer() {
|
||||||
// Proves the Referer header is attached for i.pximg.net: a header-less
|
// Proves the Referer header is attached for i.pximg.net: a header-less
|
||||||
|
|||||||
@@ -107,10 +107,60 @@ pub fn media_headers(url: &str) -> Option<Vec<(&'static str, String)>> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Flattens the app API's HTML description into plain text: `<br>` (and `<p>`)
|
||||||
|
/// become line breaks, other tags are dropped, entities decoded, the ends
|
||||||
|
/// trimmed. A caption shows text, not markup, so the author's `<a href>` links
|
||||||
|
/// contribute their link text only.
|
||||||
|
fn flatten_html(raw: &str) -> String {
|
||||||
|
let mut out = String::with_capacity(raw.len());
|
||||||
|
let mut chars = raw.chars().peekable();
|
||||||
|
while let Some(c) = chars.next() {
|
||||||
|
// Only `<` followed by `/` or a letter opens a tag — a bare `<` in
|
||||||
|
// prose ("2 < 3") is text.
|
||||||
|
let opens_tag = c == '<'
|
||||||
|
&& chars
|
||||||
|
.peek()
|
||||||
|
.is_some_and(|next| *next == '/' || next.is_ascii_alphabetic());
|
||||||
|
if !opens_tag {
|
||||||
|
out.push(c);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let mut tag = String::new();
|
||||||
|
let mut closed = false;
|
||||||
|
for c in chars.by_ref() {
|
||||||
|
if c == '>' {
|
||||||
|
closed = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
tag.push(c);
|
||||||
|
}
|
||||||
|
if !closed {
|
||||||
|
// Unclosed `<…`: keep it as text rather than dropping the tail.
|
||||||
|
out.push('<');
|
||||||
|
out.push_str(&tag);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
// `<br>`, `<br/>`, `<br />` with or without attributes, and both
|
||||||
|
// halves of a paragraph break the line; everything else is dropped.
|
||||||
|
let tag = tag
|
||||||
|
.trim()
|
||||||
|
.trim_start_matches('/')
|
||||||
|
.trim_end_matches('/')
|
||||||
|
.trim()
|
||||||
|
.to_ascii_lowercase();
|
||||||
|
if tag == "p" || tag.starts_with("br") {
|
||||||
|
out.push('\n');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
html_escape::decode_html_entities(&out).trim().to_string()
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct Illustration {
|
pub struct Illustration {
|
||||||
id: String,
|
id: String,
|
||||||
title: String,
|
title: String,
|
||||||
|
/// The artwork's description, HTML flattened to plain text.
|
||||||
|
content: String,
|
||||||
author: String,
|
author: String,
|
||||||
author_id: String,
|
author_id: String,
|
||||||
tags: Vec<String>,
|
tags: Vec<String>,
|
||||||
@@ -150,6 +200,7 @@ impl Illustration {
|
|||||||
pub fn from_model(model: &IllustrationModel) -> Self {
|
pub fn from_model(model: &IllustrationModel) -> Self {
|
||||||
let id = model.id.to_string();
|
let id = model.id.to_string();
|
||||||
let title = model.title.clone();
|
let title = model.title.clone();
|
||||||
|
let content = flatten_html(&model.caption);
|
||||||
let author = model.user.name.clone();
|
let author = model.user.name.clone();
|
||||||
let author_id = model.user.id.to_string();
|
let author_id = model.user.id.to_string();
|
||||||
let mut tags: Vec<String> = model.tags.iter().map(|tag| tag.name.clone()).collect();
|
let mut tags: Vec<String> = model.tags.iter().map(|tag| tag.name.clone()).collect();
|
||||||
@@ -193,6 +244,7 @@ impl Illustration {
|
|||||||
Self {
|
Self {
|
||||||
id,
|
id,
|
||||||
title,
|
title,
|
||||||
|
content,
|
||||||
author,
|
author,
|
||||||
author_id,
|
author_id,
|
||||||
tags,
|
tags,
|
||||||
@@ -218,12 +270,14 @@ impl From<Illustration> for Fetched {
|
|||||||
author: encode_text(&illustration.author).into_owned(),
|
author: encode_text(&illustration.author).into_owned(),
|
||||||
author_url: author_url.clone(),
|
author_url: author_url.clone(),
|
||||||
title: encode_text(&illustration.title).into_owned(),
|
title: encode_text(&illustration.title).into_owned(),
|
||||||
|
content: encode_text(&illustration.content).into_owned(),
|
||||||
tags: encode_text(&tags).into_owned(),
|
tags: encode_text(&tags).into_owned(),
|
||||||
});
|
});
|
||||||
Fetched {
|
Fetched {
|
||||||
source_url: url,
|
source_url: url,
|
||||||
caption: illustration.caption(),
|
caption: illustration.caption(),
|
||||||
title: illustration.title.clone(),
|
title: illustration.title.clone(),
|
||||||
|
content: illustration.content.clone(),
|
||||||
media: illustration.media,
|
media: illustration.media,
|
||||||
sensitive: illustration.nsfw,
|
sensitive: illustration.nsfw,
|
||||||
site_id: "pixiv",
|
site_id: "pixiv",
|
||||||
@@ -262,6 +316,7 @@ mod tests {
|
|||||||
"illust": {
|
"illust": {
|
||||||
"id": 123,
|
"id": 123,
|
||||||
"title": "Art <title>",
|
"title": "Art <title>",
|
||||||
|
"caption": "一行说明<br />二行 <a href=\"https://x.example/\">链接</a> & 结尾",
|
||||||
"type": type_,
|
"type": type_,
|
||||||
"image_urls": {
|
"image_urls": {
|
||||||
"medium": "medium.jpg",
|
"medium": "medium.jpg",
|
||||||
@@ -284,6 +339,44 @@ mod tests {
|
|||||||
Illustration::from_model(&model)
|
Illustration::from_model(&model)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The description arrives as HTML and becomes plain-text content: breaks
|
||||||
|
/// kept, tags dropped (links keep their text), entities decoded.
|
||||||
|
#[test]
|
||||||
|
fn from_json_maps_description_to_content() {
|
||||||
|
let v = illust_json("illust", 1, None, Some("o.jpg"), vec![], 0);
|
||||||
|
let illustration = parse(v);
|
||||||
|
assert_eq!(illustration.content, "一行说明\n二行 链接 & 结尾");
|
||||||
|
|
||||||
|
let fetched: Fetched = illustration.into();
|
||||||
|
assert_eq!(fetched.title, "Art <title>");
|
||||||
|
assert_eq!(fetched.content, "一行说明\n二行 链接 & 结尾");
|
||||||
|
// The built-in caption keeps its layout: the description stays out of
|
||||||
|
// it and is available through `{content}`.
|
||||||
|
assert!(!fetched.caption.contains("一行说明"), "{}", fetched.caption);
|
||||||
|
assert_eq!(
|
||||||
|
fetched.render_fields().unwrap().3,
|
||||||
|
"一行说明\n二行 链接 & 结尾"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
fetched
|
||||||
|
.caption_with("{title}: {content}")
|
||||||
|
.ends_with("一行说明\n二行 链接 & 结尾")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn flatten_html_handles_common_markup() {
|
||||||
|
assert_eq!(flatten_html(""), "");
|
||||||
|
assert_eq!(flatten_html("plain"), "plain");
|
||||||
|
assert_eq!(flatten_html("a<br />b<br/>c<br>d"), "a\nb\nc\nd");
|
||||||
|
// A paragraph break is a blank line, exactly like `<br /><br />` —
|
||||||
|
// writing it as one newline would flatten the author's paragraphs.
|
||||||
|
assert_eq!(flatten_html("<p>one</p><p>two</p>"), "one\n\ntwo");
|
||||||
|
assert_eq!(flatten_html("a & b <c>"), "a & b <c>");
|
||||||
|
// Nothing to strip: angle brackets that are not a tag survive.
|
||||||
|
assert_eq!(flatten_html("2 < 3"), "2 < 3");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn pattern_matches_all_forms() {
|
fn pattern_matches_all_forms() {
|
||||||
let cases = [
|
let cases = [
|
||||||
|
|||||||
@@ -6,6 +6,10 @@ use serde::Deserialize;
|
|||||||
pub struct IllustrationModel {
|
pub struct IllustrationModel {
|
||||||
pub id: u64,
|
pub id: u64,
|
||||||
pub title: String,
|
pub title: String,
|
||||||
|
/// The artwork's description as the app API returns it — HTML in most
|
||||||
|
/// works (`<br />`, `<a href>`, sometimes `<p>`), empty for many.
|
||||||
|
#[serde(default)]
|
||||||
|
pub caption: String,
|
||||||
pub r#type: TypeModel,
|
pub r#type: TypeModel,
|
||||||
pub image_urls: ImageUrlsModel,
|
pub image_urls: ImageUrlsModel,
|
||||||
pub user: UserInfoModel,
|
pub user: UserInfoModel,
|
||||||
|
|||||||
@@ -327,7 +327,8 @@ mod tests {
|
|||||||
"https://x.com/nsfw_author/status/2083868672721039569"
|
"https://x.com/nsfw_author/status/2083868672721039569"
|
||||||
);
|
);
|
||||||
// The appended media short link (no URL-entity mapping) is stripped.
|
// The appended media short link (no URL-entity mapping) is stripped.
|
||||||
assert_eq!(fetched.title, "nsfw content");
|
assert_eq!(fetched.title, "");
|
||||||
|
assert_eq!(fetched.content, "nsfw content");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -43,27 +43,25 @@ pub async fn fetch_from_url(url: &str) -> Result<Fetched, FetchError> {
|
|||||||
match fetch(id).await {
|
match fetch(id).await {
|
||||||
Ok(tweet) => Ok(tweet.into()),
|
Ok(tweet) => Ok(tweet.into()),
|
||||||
// Syndication withholds NSFW/age-restricted tweets (empty `{}`).
|
// Syndication withholds NSFW/age-restricted tweets (empty `{}`).
|
||||||
// Retry as the logged-in user when TWITTER_AUTH_TOKEN is set;
|
// Retry as the logged-in user when TWITTER_AUTH_TOKEN is set; without
|
||||||
// otherwise degrade to an empty result (the bot replies
|
// the token the withholding is reported as `Sensitive`, so the bot can
|
||||||
// "No media found").
|
// answer "age-restricted / needs TWITTER_AUTH_TOKEN" instead of the
|
||||||
|
// misleading "No media found".
|
||||||
Err(FetchError::Sensitive) => {
|
Err(FetchError::Sensitive) => {
|
||||||
if super::auth::enabled() {
|
if super::auth::enabled() {
|
||||||
match super::auth::fetch(id).await {
|
match super::auth::fetch(id).await {
|
||||||
Ok(tweet) => Ok(tweet.into()),
|
Ok(tweet) => Ok(tweet.into()),
|
||||||
// The tweet is genuinely gone (deleted / suspended /
|
// Deleted/suspended (tombstoned) and unexpected fallback
|
||||||
// tombstoned): report it instead of degrading to an
|
// failures keep their own class: the bot reports what
|
||||||
// empty result ("No media found"). Only unexpected
|
// actually happened rather than "No media found".
|
||||||
// fallback failures (network, parse) keep the NSFW
|
|
||||||
// placeholder.
|
|
||||||
Err(FetchError::NotFound) => Err(FetchError::NotFound),
|
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
log::warn!("twitter auth fallback failed for {id}: {e}");
|
log::warn!("twitter auth fallback failed for {id}: {e}");
|
||||||
Ok(empty_fetched(url))
|
Err(e)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
log::debug!("tweet {id} is sensitive; set TWITTER_AUTH_TOKEN to fetch NSFW media");
|
log::debug!("tweet {id} is sensitive; set TWITTER_AUTH_TOKEN to fetch NSFW media");
|
||||||
Ok(empty_fetched(url))
|
Err(FetchError::Sensitive)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(e) => Err(e),
|
Err(e) => Err(e),
|
||||||
@@ -90,23 +88,6 @@ pub fn media_headers(_url: &str) -> Option<Vec<(&'static str, String)>> {
|
|||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A Fetched with no media for withheld tweets: the bot replies
|
|
||||||
/// "No media found" and moves on instead of erroring.
|
|
||||||
fn empty_fetched(url: &str) -> Fetched {
|
|
||||||
Fetched {
|
|
||||||
source_url: url.to_string(),
|
|
||||||
// The raw user-supplied URL goes into an HTML caption; escape it so
|
|
||||||
// crafted links cannot break the parse (Telegram 400).
|
|
||||||
caption: encode_text(url).into_owned(),
|
|
||||||
title: String::new(),
|
|
||||||
media: vec![],
|
|
||||||
sensitive: true,
|
|
||||||
site_id: "twitter",
|
|
||||||
render_data: None,
|
|
||||||
_keep_alive: None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Fetches a tweet from the syndication endpoint. Deleted/blocked tweets
|
/// Fetches a tweet from the syndication endpoint. Deleted/blocked tweets
|
||||||
/// surface as `FetchError::NotFound`; withheld content (empty tombstone,
|
/// surface as `FetchError::NotFound`; withheld content (empty tombstone,
|
||||||
/// age-restricted) as `FetchError::Sensitive`.
|
/// age-restricted) as `FetchError::Sensitive`.
|
||||||
@@ -352,17 +333,20 @@ impl From<Tweet> for Fetched {
|
|||||||
fn from(tweet: Tweet) -> Self {
|
fn from(tweet: Tweet) -> Self {
|
||||||
let url = tweet.url();
|
let url = tweet.url();
|
||||||
let author_url = tweet.author_url();
|
let author_url = tweet.author_url();
|
||||||
|
// A tweet has no title: its text is all content.
|
||||||
let render_data = Some(crate::site::RenderData {
|
let render_data = Some(crate::site::RenderData {
|
||||||
url: url.clone(),
|
url: url.clone(),
|
||||||
author: encode_text(&tweet.author).into_owned(),
|
author: encode_text(&tweet.author).into_owned(),
|
||||||
author_url: author_url.clone(),
|
author_url: author_url.clone(),
|
||||||
title: encode_text(&tweet.text).into_owned(),
|
title: String::new(),
|
||||||
|
content: encode_text(&tweet.text).into_owned(),
|
||||||
tags: String::new(),
|
tags: String::new(),
|
||||||
});
|
});
|
||||||
Fetched {
|
Fetched {
|
||||||
source_url: url,
|
source_url: url,
|
||||||
caption: tweet.caption(),
|
caption: tweet.caption(),
|
||||||
title: tweet.text.clone(),
|
title: String::new(),
|
||||||
|
content: tweet.text.clone(),
|
||||||
media: tweet.media,
|
media: tweet.media,
|
||||||
sensitive: tweet.sensitive,
|
sensitive: tweet.sensitive,
|
||||||
site_id: "twitter",
|
site_id: "twitter",
|
||||||
@@ -437,7 +421,8 @@ mod tests {
|
|||||||
assert_eq!(tweet.text, ">^ω^< & more 'quoted'");
|
assert_eq!(tweet.text, ">^ω^< & more 'quoted'");
|
||||||
assert_eq!(tweet.author, "O'Brien");
|
assert_eq!(tweet.author, "O'Brien");
|
||||||
let fetched: Fetched = tweet.into();
|
let fetched: Fetched = tweet.into();
|
||||||
assert_eq!(fetched.title, ">^ω^< & more 'quoted'");
|
assert_eq!(fetched.title, "");
|
||||||
|
assert_eq!(fetched.content, ">^ω^< & more 'quoted'");
|
||||||
// The caption escapes the raw text exactly once (encode_text covers
|
// The caption escapes the raw text exactly once (encode_text covers
|
||||||
// & < >; apostrophes stay literal — they are harmless in text).
|
// & < >; apostrophes stay literal — they are harmless in text).
|
||||||
assert!(
|
assert!(
|
||||||
@@ -496,7 +481,8 @@ mod tests {
|
|||||||
fetched.source_url,
|
fetched.source_url,
|
||||||
"https://x.com/author_handle/status/861627479294746624"
|
"https://x.com/author_handle/status/861627479294746624"
|
||||||
);
|
);
|
||||||
assert_eq!(fetched.title, "a & b <c>");
|
assert_eq!(fetched.title, "");
|
||||||
|
assert_eq!(fetched.content, "a & b <c>");
|
||||||
assert!(fetched.sensitive);
|
assert!(fetched.sensitive);
|
||||||
assert_eq!(fetched.media.len(), 2);
|
assert_eq!(fetched.media.len(), 2);
|
||||||
match &fetched.media[0] {
|
match &fetched.media[0] {
|
||||||
|
|||||||
@@ -1,11 +1,11 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "xmedia-bot"
|
name = "xmedia-bot"
|
||||||
version = "1.5.0"
|
version = "1.8.0"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
teloxide = { version = "0.17", default-features = false, features = ["webhooks-axum", "macros", "rustls", "ctrlc_handler"] }
|
teloxide = { version = "0.17", default-features = false, features = ["webhooks-axum", "macros", "rustls", "ctrlc_handler"] }
|
||||||
tokio = { version = "1.40", features = ["rt-multi-thread", "macros", "time"] }
|
tokio = { version = "1.40", features = ["rt-multi-thread", "macros", "time", "sync"] }
|
||||||
serde = { version = "1", features = ["derive"] }
|
serde = { version = "1", features = ["derive"] }
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
log = "0.4"
|
log = "0.4"
|
||||||
@@ -13,8 +13,8 @@ pretty_env_logger = "0.5"
|
|||||||
dotenv = "0.15"
|
dotenv = "0.15"
|
||||||
url = "2.5.2"
|
url = "2.5.2"
|
||||||
html-escape = "0.2"
|
html-escape = "0.2"
|
||||||
rusqlite = { version = "0.32", features = ["bundled"] }
|
rusqlite = { version = "0.40", features = ["bundled"] }
|
||||||
rand = "0.8"
|
rand = "0.10"
|
||||||
tempfile = "3"
|
tempfile = "3"
|
||||||
parking_lot = "0.12"
|
parking_lot = "0.12"
|
||||||
bytes = "1"
|
bytes = "1"
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
//! Central env handling. The only other places that read env are
|
//! Central env handling. The only other places that read env are
|
||||||
//! `Bot::from_env` (TELOXIDE_TOKEN) and x-media (PIXIV_REFRESH_TOKEN).
|
//! `Bot::from_env` (TELOXIDE_TOKEN) and x-media (PIXIV_REFRESH_TOKEN,
|
||||||
|
//! TWITTER_AUTH_TOKEN, BILIBILI_COOKIE).
|
||||||
|
|
||||||
use std::env;
|
use std::env;
|
||||||
use std::net::IpAddr;
|
use std::net::IpAddr;
|
||||||
@@ -12,6 +13,10 @@ pub struct Config {
|
|||||||
pub edit_message_ttl: Duration,
|
pub edit_message_ttl: Duration,
|
||||||
/// LINK_CACHE_TTL_SECONDS, default 604800 (7 days).
|
/// LINK_CACHE_TTL_SECONDS, default 604800 (7 days).
|
||||||
pub link_cache_ttl: Duration,
|
pub link_cache_ttl: Duration,
|
||||||
|
/// CAPTION_QUOTE_TEXT_CHARS, default 200: a post whose text (title plus
|
||||||
|
/// content) is at least this many characters gets that text wrapped in an
|
||||||
|
/// expandable blockquote inside its caption. `0` disables the wrap.
|
||||||
|
pub caption_quote_text_chars: usize,
|
||||||
// Webhook settings (moved out of main; names/defaults unchanged).
|
// Webhook settings (moved out of main; names/defaults unchanged).
|
||||||
pub webhook_enabled: bool,
|
pub webhook_enabled: bool,
|
||||||
pub webhook_url: Option<url::Url>,
|
pub webhook_url: Option<url::Url>,
|
||||||
@@ -57,6 +62,7 @@ impl Config {
|
|||||||
Duration::from_secs(parse_u64("EDIT_MESSAGE_TTL_SECONDS", 24 * 3600));
|
Duration::from_secs(parse_u64("EDIT_MESSAGE_TTL_SECONDS", 24 * 3600));
|
||||||
let link_cache_ttl =
|
let link_cache_ttl =
|
||||||
Duration::from_secs(parse_u64("LINK_CACHE_TTL_SECONDS", 7 * 24 * 3600));
|
Duration::from_secs(parse_u64("LINK_CACHE_TTL_SECONDS", 7 * 24 * 3600));
|
||||||
|
let caption_quote_text_chars = parse_u64("CAPTION_QUOTE_TEXT_CHARS", 200) as usize;
|
||||||
|
|
||||||
let webhook_enabled = env::var("WEBHOOK")
|
let webhook_enabled = env::var("WEBHOOK")
|
||||||
.is_ok_and(|v| matches!(v.to_lowercase().as_str(), "true" | "yes" | "1"));
|
.is_ok_and(|v| matches!(v.to_lowercase().as_str(), "true" | "yes" | "1"));
|
||||||
@@ -92,6 +98,7 @@ impl Config {
|
|||||||
admin_ids,
|
admin_ids,
|
||||||
edit_message_ttl,
|
edit_message_ttl,
|
||||||
link_cache_ttl,
|
link_cache_ttl,
|
||||||
|
caption_quote_text_chars,
|
||||||
webhook_enabled,
|
webhook_enabled,
|
||||||
webhook_url,
|
webhook_url,
|
||||||
webhook_listen,
|
webhook_listen,
|
||||||
|
|||||||
@@ -0,0 +1,124 @@
|
|||||||
|
//! Runtime context: the collaborators a handler needs, injected as one struct
|
||||||
|
//! so tests can substitute a scripted sender and tempdir-backed stores.
|
||||||
|
//!
|
||||||
|
//! The production context is assembled from the process-wide statics
|
||||||
|
//! ([`AppContext::from_statics`]); the spawned worker closures hold
|
||||||
|
//! [`CONTEXT`], which is `'static` for that reason.
|
||||||
|
|
||||||
|
use crate::config::Config;
|
||||||
|
use crate::handlers::{CHAT_STORE, CONFIG, LINK_CACHE, TASK_QUEUE};
|
||||||
|
use crate::link_cache::LinkCache;
|
||||||
|
use crate::media_sender::MediaSender;
|
||||||
|
use crate::queue::PersistentTaskQueue;
|
||||||
|
use crate::send::BOT;
|
||||||
|
use crate::state::ChatStore;
|
||||||
|
use std::sync::LazyLock;
|
||||||
|
|
||||||
|
pub struct AppContext<'a> {
|
||||||
|
pub sender: &'a dyn MediaSender,
|
||||||
|
pub chat_store: &'a ChatStore,
|
||||||
|
pub task_queue: &'a PersistentTaskQueue,
|
||||||
|
pub link_cache: &'a LinkCache,
|
||||||
|
pub config: &'a Config,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a> AppContext<'a> {
|
||||||
|
/// The stores are the process-wide statics; `sender` is whatever the caller
|
||||||
|
/// was handed (the dispatcher's `Bot` clone for update handlers, the shared
|
||||||
|
/// queue `Bot` for the worker loops). Update handlers build their own
|
||||||
|
/// context from the `Bot` they received so the same code path works with an
|
||||||
|
/// injected mock in tests.
|
||||||
|
pub fn from_statics(sender: &'a dyn MediaSender) -> AppContext<'a> {
|
||||||
|
AppContext {
|
||||||
|
sender,
|
||||||
|
chat_store: &CHAT_STORE,
|
||||||
|
task_queue: &TASK_QUEUE,
|
||||||
|
link_cache: &LINK_CACHE,
|
||||||
|
config: &CONFIG,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The URL/queue workers' context: `'static` because `tokio::spawn`ed closures
|
||||||
|
/// and the queue's handler type require it.
|
||||||
|
pub static CONTEXT: LazyLock<AppContext<'static>> =
|
||||||
|
LazyLock::new(|| AppContext::from_statics(&*BOT));
|
||||||
|
|
||||||
|
/// Test support: a tempdir-backed set of stores plus the context borrowing
|
||||||
|
/// them, so a handler test needs one line of setup.
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) mod test_support {
|
||||||
|
use super::*;
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
pub(crate) struct TestStores {
|
||||||
|
_dir: tempfile::TempDir,
|
||||||
|
pool: Arc<crate::db::DbPool>,
|
||||||
|
chat_store: ChatStore,
|
||||||
|
task_queue: PersistentTaskQueue,
|
||||||
|
link_cache: LinkCache,
|
||||||
|
config: Config,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TestStores {
|
||||||
|
pub(crate) fn new() -> Self {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let pool = crate::db::open_store(dir.path().join("ctx.db").to_str().unwrap()).unwrap();
|
||||||
|
TestStores {
|
||||||
|
_dir: dir,
|
||||||
|
chat_store: ChatStore::new(Arc::clone(&pool)),
|
||||||
|
task_queue: PersistentTaskQueue::new(Arc::clone(&pool)),
|
||||||
|
link_cache: LinkCache::new(Arc::clone(&pool)),
|
||||||
|
config: Config::load(),
|
||||||
|
pool,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn ctx<'a>(&'a self, sender: &'a dyn MediaSender) -> AppContext<'a> {
|
||||||
|
AppContext {
|
||||||
|
sender,
|
||||||
|
chat_store: &self.chat_store,
|
||||||
|
task_queue: &self.task_queue,
|
||||||
|
link_cache: &self.link_cache,
|
||||||
|
config: &self.config,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn chat_store(&self) -> &ChatStore {
|
||||||
|
&self.chat_store
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The parsed config, mutable so a test can pin a knob (e.g. the
|
||||||
|
/// caption-quote threshold) instead of depending on the environment.
|
||||||
|
pub(crate) fn config_mut(&mut self) -> &mut Config {
|
||||||
|
&mut self.config
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn link_cache(&self) -> &LinkCache {
|
||||||
|
&self.link_cache
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Rows persisted in the task queue: what "queued for retry" looks like
|
||||||
|
/// from the outside.
|
||||||
|
pub(crate) async fn queued_tasks(&self) -> i64 {
|
||||||
|
let pool = Arc::clone(&self.pool);
|
||||||
|
pool.with_conn(|conn| {
|
||||||
|
conn.query_row("SELECT COUNT(*) FROM tasks", [], |row| row.get(0))
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The single queued task payload, for asserting what was rescheduled.
|
||||||
|
pub(crate) async fn queued_payload(&self) -> serde_json::Value {
|
||||||
|
let pool = Arc::clone(&self.pool);
|
||||||
|
let payload: String = pool
|
||||||
|
.with_conn(|conn| {
|
||||||
|
conn.query_row("SELECT payload FROM tasks LIMIT 1", [], |row| row.get(0))
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
serde_json::from_str(&payload).unwrap()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,56 +1,101 @@
|
|||||||
//! Callback query handling: the edit-before-forward prompt's "forward" and
|
//! Callback query handling: the edit-before-forward prompt's `"forward"` and
|
||||||
//! "template|<name>" buttons.
|
//! `"template|<name>"` buttons.
|
||||||
|
//!
|
||||||
|
//! [`callback_query_handler`] is the dptree entry; it only pulls the plain
|
||||||
|
//! values out of the teloxide update and hands them to [`handle_callback`],
|
||||||
|
//! which holds the button logic and is driven directly by tests.
|
||||||
|
|
||||||
use super::urls::enqueue_retry;
|
use crate::ctx::AppContext;
|
||||||
use super::{CHAT_STORE, CONFIG, TASK_QUEUE};
|
|
||||||
use crate::db::unix_now;
|
use crate::db::unix_now;
|
||||||
use crate::send::{self, Task};
|
use crate::send::{self, Task};
|
||||||
use teloxide::RequestError;
|
use teloxide::RequestError;
|
||||||
use teloxide::prelude::*;
|
use teloxide::prelude::*;
|
||||||
use teloxide::types::{CallbackQuery, ChatId, MessageId, ParseMode};
|
use teloxide::types::{CallbackQuery, CallbackQueryId, MessageId};
|
||||||
|
|
||||||
|
/// The `"forward"` button's data.
|
||||||
|
const FORWARD: &str = "forward";
|
||||||
|
/// The `"skip"` button's data: drop the prompt without forwarding.
|
||||||
|
const SKIP: &str = "skip";
|
||||||
|
/// Prefix of a template button's data: `"template|<name>"`.
|
||||||
|
const TEMPLATE_PREFIX: &str = "template|";
|
||||||
|
|
||||||
pub async fn callback_query_handler(bot: Bot, query: CallbackQuery) -> Result<(), RequestError> {
|
pub async fn callback_query_handler(bot: Bot, query: CallbackQuery) -> Result<(), RequestError> {
|
||||||
let callback_query_id = query.id;
|
|
||||||
let data = query.data.clone();
|
|
||||||
let Some(message) = &query.message else {
|
let Some(message) = &query.message else {
|
||||||
return respond(());
|
return respond(());
|
||||||
};
|
};
|
||||||
let chat_id = message.chat().id.0;
|
let Some(data) = query.data.clone() else {
|
||||||
let prompt_message_id = message.id().0 as i64;
|
return respond(());
|
||||||
let ttl_secs = CONFIG.edit_message_ttl.as_secs() as i64;
|
};
|
||||||
let chat_data = CHAT_STORE.get(chat_id).await;
|
let ctx = AppContext::from_statics(&bot);
|
||||||
|
handle_callback(
|
||||||
|
&ctx,
|
||||||
|
query.id.clone(),
|
||||||
|
message.chat().id.0,
|
||||||
|
message.id().0 as i64,
|
||||||
|
&data,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
respond(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Handles one button press on the edit-before-forward prompt.
|
||||||
|
async fn handle_callback(
|
||||||
|
ctx: &AppContext<'_>,
|
||||||
|
callback_query_id: CallbackQueryId,
|
||||||
|
chat_id: i64,
|
||||||
|
prompt_message_id: i64,
|
||||||
|
data: &str,
|
||||||
|
) {
|
||||||
|
let ttl_secs = ctx.config.edit_message_ttl.as_secs() as i64;
|
||||||
|
let chat_data = ctx.chat_store.get(chat_id).await;
|
||||||
let edit = chat_data.edit_message.get(&prompt_message_id).cloned();
|
let edit = chat_data.edit_message.get(&prompt_message_id).cloned();
|
||||||
let Some(edit) = edit else {
|
let Some(edit) = edit else {
|
||||||
log::debug!(
|
log::debug!("callback from {chat_id}: no edit record for prompt {prompt_message_id}");
|
||||||
"callback from {}: no edit record for prompt {prompt_message_id}",
|
let _ = ctx
|
||||||
chat_id
|
.sender
|
||||||
);
|
.answer_callback_query(callback_query_id, Some("Expired".to_string()))
|
||||||
bot.answer_callback_query(callback_query_id)
|
.await;
|
||||||
.text("Expired")
|
return;
|
||||||
.await?;
|
|
||||||
return respond(());
|
|
||||||
};
|
};
|
||||||
// Lazy expiry: a stale record (past the TTL, not yet swept) is dropped.
|
// Lazy expiry: a stale record (past the TTL, not yet swept) is dropped.
|
||||||
if edit.created_at + ttl_secs <= unix_now() {
|
if edit.created_at + ttl_secs <= unix_now() {
|
||||||
CHAT_STORE
|
ctx.chat_store
|
||||||
.update(chat_id, |data| {
|
.update(chat_id, |data| {
|
||||||
data.edit_message.remove(&prompt_message_id);
|
data.edit_message.remove(&prompt_message_id);
|
||||||
})
|
})
|
||||||
.await;
|
.await;
|
||||||
bot.answer_callback_query(callback_query_id)
|
let _ = ctx
|
||||||
.text("Expired")
|
.sender
|
||||||
.await?;
|
.answer_callback_query(callback_query_id, Some("Expired".to_string()))
|
||||||
return respond(());
|
.await;
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
let Some(data) = data else {
|
log::info!("callback from {chat_id} on prompt {prompt_message_id}: {data}");
|
||||||
return respond(());
|
if data == SKIP {
|
||||||
};
|
// Skip works with or without a forward channel: it is the explicit
|
||||||
log::info!(
|
// "do not forward this" answer, and it drops the record so the forward
|
||||||
"callback from {} on prompt {prompt_message_id}: {data}",
|
// can never happen later.
|
||||||
chat_id
|
log::info!("edit-before-forward prompt {prompt_message_id} skipped");
|
||||||
);
|
ctx.chat_store
|
||||||
if data == "forward" {
|
.update(chat_id, |data| {
|
||||||
|
data.edit_message.remove(&prompt_message_id);
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
let _ = ctx
|
||||||
|
.sender
|
||||||
|
.delete_message(ChatId(chat_id), MessageId(prompt_message_id as i32))
|
||||||
|
.await;
|
||||||
|
let _ = ctx
|
||||||
|
.sender
|
||||||
|
.answer_callback_query(
|
||||||
|
callback_query_id,
|
||||||
|
Some("Skipped — nothing was forwarded.".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if data == FORWARD {
|
||||||
match chat_data.forward_channel_id {
|
match chat_data.forward_channel_id {
|
||||||
Some(channel_id) => {
|
Some(channel_id) => {
|
||||||
let forward_task = Task::ForwardMessages {
|
let forward_task = Task::ForwardMessages {
|
||||||
@@ -60,62 +105,72 @@ pub async fn callback_query_handler(bot: Bot, query: CallbackQuery) -> Result<()
|
|||||||
notify_chat_id: Some(chat_id),
|
notify_chat_id: Some(chat_id),
|
||||||
notify_message_id: Some(prompt_message_id),
|
notify_message_id: Some(prompt_message_id),
|
||||||
};
|
};
|
||||||
match send::forward_messages(&bot, &forward_task).await {
|
let (answer, settled) = match send::forward_messages(ctx, &forward_task).await {
|
||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
log::info!(
|
log::info!(
|
||||||
"forwarded {} message(s) to channel {channel_id}",
|
"forwarded {} message(s) to channel {channel_id}",
|
||||||
edit.forward_message_ids.len()
|
edit.forward_message_ids.len()
|
||||||
);
|
);
|
||||||
bot.answer_callback_query(callback_query_id)
|
("✅ Forwarded".to_string(), true)
|
||||||
.text("✅ Forwarded")
|
|
||||||
.await?;
|
|
||||||
let _ = bot
|
|
||||||
.delete_message(ChatId(chat_id), MessageId(prompt_message_id as i32))
|
|
||||||
.await;
|
|
||||||
CHAT_STORE
|
|
||||||
.update(chat_id, |data| {
|
|
||||||
data.edit_message.remove(&prompt_message_id);
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
}
|
}
|
||||||
Err(send::SendError::Retryable {
|
Err(send::SendError::Retryable {
|
||||||
delay_seconds,
|
delay_seconds,
|
||||||
task,
|
task,
|
||||||
}) => {
|
}) => {
|
||||||
log::info!("forward queued for retry in {delay_seconds:.1}s");
|
log::info!("forward queued for retry in {delay_seconds:.1}s");
|
||||||
enqueue_retry(&TASK_QUEUE, *task, delay_seconds).await;
|
send::enqueue_retry(ctx.task_queue, *task, delay_seconds).await;
|
||||||
bot.answer_callback_query(callback_query_id)
|
("Forward queued for retry.".to_string(), false)
|
||||||
.text("Forward queued for retry.")
|
|
||||||
.await?;
|
|
||||||
}
|
}
|
||||||
Err(send::SendError::Permanent { message, .. }) => {
|
Err(send::SendError::Permanent { message, .. }) => {
|
||||||
log::error!("forward failed permanently: {message}");
|
log::error!("forward failed permanently: {message}");
|
||||||
bot.answer_callback_query(callback_query_id)
|
(format!("Forward failed: {message}"), false)
|
||||||
.text(format!("Forward failed: {message}"))
|
|
||||||
.await?;
|
|
||||||
}
|
}
|
||||||
|
};
|
||||||
|
if settled {
|
||||||
|
// The prompt is done: drop it and its record.
|
||||||
|
let _ = ctx
|
||||||
|
.sender
|
||||||
|
.delete_message(ChatId(chat_id), MessageId(prompt_message_id as i32))
|
||||||
|
.await;
|
||||||
|
ctx.chat_store
|
||||||
|
.update(chat_id, |data| {
|
||||||
|
data.edit_message.remove(&prompt_message_id);
|
||||||
|
})
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
|
let _ = ctx
|
||||||
|
.sender
|
||||||
|
.answer_callback_query(callback_query_id, Some(answer))
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
None => {
|
None => {
|
||||||
log::debug!("forward callback without a forward channel set");
|
log::debug!("forward callback without a forward channel set");
|
||||||
bot.answer_callback_query(callback_query_id)
|
let _ = ctx
|
||||||
.text("No forward channel set.")
|
.sender
|
||||||
.await?;
|
.answer_callback_query(
|
||||||
|
callback_query_id,
|
||||||
|
Some("No forward channel set.".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return respond(());
|
return;
|
||||||
}
|
}
|
||||||
if let Some(name) = data.strip_prefix("template|") {
|
|
||||||
|
if let Some(name) = data.strip_prefix(TEMPLATE_PREFIX) {
|
||||||
if let Some(template_html) = chat_data.template.get(name).cloned()
|
if let Some(template_html) = chat_data.template.get(name).cloned()
|
||||||
&& let Some(first_forward_id) = edit.forward_message_ids.first().copied()
|
&& let Some(first_forward_id) = edit.forward_message_ids.first().copied()
|
||||||
{
|
{
|
||||||
// Raw template including the [] placeholder (Python parity).
|
// Raw template including the [] placeholder (Python parity).
|
||||||
let _ = bot
|
let _ = ctx
|
||||||
.edit_message_caption(ChatId(chat_id), MessageId(first_forward_id as i32))
|
.sender
|
||||||
.caption(template_html)
|
.edit_message_caption(
|
||||||
.parse_mode(ParseMode::Html)
|
ChatId(chat_id),
|
||||||
|
MessageId(first_forward_id as i32),
|
||||||
|
template_html,
|
||||||
|
)
|
||||||
.await;
|
.await;
|
||||||
CHAT_STORE
|
ctx.chat_store
|
||||||
.update(chat_id, |data| {
|
.update(chat_id, |data| {
|
||||||
if let Some(entry) = data.edit_message.get_mut(&prompt_message_id) {
|
if let Some(entry) = data.edit_message.get_mut(&prompt_message_id) {
|
||||||
entry.template = name.to_string();
|
entry.template = name.to_string();
|
||||||
@@ -124,7 +179,193 @@ pub async fn callback_query_handler(bot: Bot, query: CallbackQuery) -> Result<()
|
|||||||
.await;
|
.await;
|
||||||
log::info!("template '{name}' applied to prompt {prompt_message_id}");
|
log::info!("template '{name}' applied to prompt {prompt_message_id}");
|
||||||
}
|
}
|
||||||
bot.answer_callback_query(callback_query_id).await?;
|
let _ = ctx
|
||||||
|
.sender
|
||||||
|
.answer_callback_query(callback_query_id, None)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::ctx::test_support::TestStores;
|
||||||
|
use crate::media_sender::test_support::{MockSender, Outcome};
|
||||||
|
use crate::state::EditMessage;
|
||||||
|
use teloxide::ApiError;
|
||||||
|
|
||||||
|
/// The edit-before-forward prompt's message id in these tests.
|
||||||
|
const PROMPT_ID: i64 = 7;
|
||||||
|
/// The message the prompt refers to (the one whose caption is swapped).
|
||||||
|
const FORWARDED_ID: i64 = 9;
|
||||||
|
|
||||||
|
fn api_error() -> RequestError {
|
||||||
|
RequestError::Api(ApiError::Unknown("Bad Request: chat not found".into()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn callback_id() -> CallbackQueryId {
|
||||||
|
CallbackQueryId("cb-1".to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Seeds a live prompt record plus a forward channel and a template;
|
||||||
|
/// `created_at` backdates the record for the expiry cases.
|
||||||
|
async fn seed_prompt(ctx: &AppContext<'_>, created_at: i64) {
|
||||||
|
ctx.chat_store
|
||||||
|
.update(1, |data| {
|
||||||
|
data.forward_channel_id = Some(2);
|
||||||
|
data.template
|
||||||
|
.insert("tpl".to_string(), "<b>[]</b>".to_string());
|
||||||
|
data.edit_message.insert(
|
||||||
|
PROMPT_ID,
|
||||||
|
EditMessage {
|
||||||
|
url: "https://x.com/u/status/1".into(),
|
||||||
|
chat_id: 1,
|
||||||
|
forward_message_ids: vec![FORWARDED_ID],
|
||||||
|
template: String::new(),
|
||||||
|
created_at,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn template_button_swaps_the_caption_and_records_the_choice() {
|
||||||
|
let sender = MockSender::scripted(vec![Outcome::EditOk], api_error);
|
||||||
|
let stores = TestStores::new();
|
||||||
|
let ctx = stores.ctx(&sender);
|
||||||
|
seed_prompt(&ctx, crate::db::unix_now()).await;
|
||||||
|
|
||||||
|
handle_callback(&ctx, callback_id(), 1, PROMPT_ID, "template|tpl").await;
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
sender.calls(),
|
||||||
|
vec!["edit_message_caption", "answer_callback_query"]
|
||||||
|
);
|
||||||
|
// The raw template, including the [] the user edits into.
|
||||||
|
assert_eq!(sender.captions(), vec!["<b>[]</b>"]);
|
||||||
|
assert_eq!(sender.answers(), vec![None]);
|
||||||
|
let data = ctx.chat_store.get(1).await;
|
||||||
|
assert_eq!(data.edit_message[&PROMPT_ID].template, "tpl");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn forward_button_copies_then_clears_the_prompt() {
|
||||||
|
let sender = MockSender::scripted(vec![Outcome::CopyOk], api_error);
|
||||||
|
let stores = TestStores::new();
|
||||||
|
let ctx = stores.ctx(&sender);
|
||||||
|
seed_prompt(&ctx, crate::db::unix_now()).await;
|
||||||
|
|
||||||
|
handle_callback(&ctx, callback_id(), 1, PROMPT_ID, "forward").await;
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
sender.calls(),
|
||||||
|
vec!["copy_messages", "delete_message", "answer_callback_query"]
|
||||||
|
);
|
||||||
|
assert_eq!(sender.answers(), vec![Some("✅ Forwarded".to_string())]);
|
||||||
|
assert!(
|
||||||
|
ctx.chat_store.get(1).await.edit_message.is_empty(),
|
||||||
|
"a settled prompt must drop its record"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn skip_drops_the_prompt_without_forwarding() {
|
||||||
|
// "skip" needs no forward channel and no scripted outcomes: it deletes
|
||||||
|
// the prompt and drops the record, so no forward can ever happen.
|
||||||
|
let sender = MockSender::scripted(vec![], api_error);
|
||||||
|
let stores = TestStores::new();
|
||||||
|
let ctx = stores.ctx(&sender);
|
||||||
|
seed_prompt(&ctx, crate::db::unix_now()).await;
|
||||||
|
|
||||||
|
handle_callback(&ctx, callback_id(), 1, PROMPT_ID, "skip").await;
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
sender.calls(),
|
||||||
|
vec!["delete_message", "answer_callback_query"]
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
sender.answers(),
|
||||||
|
vec![Some("Skipped — nothing was forwarded.".to_string())]
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
ctx.chat_store.get(1).await.edit_message.is_empty(),
|
||||||
|
"a skipped prompt must drop its record"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn forward_without_a_channel_is_reported() {
|
||||||
|
let sender = MockSender::scripted(vec![], api_error);
|
||||||
|
let stores = TestStores::new();
|
||||||
|
let ctx = stores.ctx(&sender);
|
||||||
|
seed_prompt(&ctx, crate::db::unix_now()).await;
|
||||||
|
ctx.chat_store
|
||||||
|
.update(1, |data| data.forward_channel_id = None)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
handle_callback(&ctx, callback_id(), 1, PROMPT_ID, "forward").await;
|
||||||
|
|
||||||
|
assert_eq!(sender.calls(), vec!["answer_callback_query"]);
|
||||||
|
assert_eq!(
|
||||||
|
sender.answers(),
|
||||||
|
vec![Some("No forward channel set.".to_string())]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn retryable_forward_is_queued_and_keeps_the_prompt() {
|
||||||
|
use teloxide::types::Seconds;
|
||||||
|
let sender = MockSender::scripted(vec![Outcome::CopyErr], || {
|
||||||
|
RequestError::RetryAfter(Seconds::from_seconds(7))
|
||||||
|
});
|
||||||
|
let stores = TestStores::new();
|
||||||
|
let ctx = stores.ctx(&sender);
|
||||||
|
seed_prompt(&ctx, crate::db::unix_now()).await;
|
||||||
|
|
||||||
|
handle_callback(&ctx, callback_id(), 1, PROMPT_ID, "forward").await;
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
sender.calls(),
|
||||||
|
vec!["copy_messages", "answer_callback_query"]
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
sender.answers(),
|
||||||
|
vec![Some("Forward queued for retry.".to_string())]
|
||||||
|
);
|
||||||
|
assert_eq!(stores.queued_tasks().await, 1);
|
||||||
|
// The prompt is not settled: the queued retry still needs the record.
|
||||||
|
assert!(
|
||||||
|
ctx.chat_store
|
||||||
|
.get(1)
|
||||||
|
.await
|
||||||
|
.edit_message
|
||||||
|
.contains_key(&PROMPT_ID)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn unknown_and_expired_prompts_answer_expired() {
|
||||||
|
let sender = MockSender::scripted(vec![], api_error);
|
||||||
|
let stores = TestStores::new();
|
||||||
|
let ctx = stores.ctx(&sender);
|
||||||
|
|
||||||
|
// No record at all.
|
||||||
|
handle_callback(&ctx, callback_id(), 1, PROMPT_ID, "forward").await;
|
||||||
|
assert_eq!(sender.answers(), vec![Some("Expired".to_string())]);
|
||||||
|
|
||||||
|
// A record past its TTL (nothing swept it yet) is dropped on use.
|
||||||
|
let stale = crate::db::unix_now() - ctx.config.edit_message_ttl.as_secs() as i64 - 1;
|
||||||
|
seed_prompt(&ctx, stale).await;
|
||||||
|
handle_callback(&ctx, callback_id(), 1, PROMPT_ID, "forward").await;
|
||||||
|
assert_eq!(
|
||||||
|
sender.answers(),
|
||||||
|
vec![Some("Expired".to_string()), Some("Expired".to_string())]
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
ctx.chat_store.get(1).await.edit_message.is_empty(),
|
||||||
|
"the expired record must be dropped"
|
||||||
|
);
|
||||||
|
assert_eq!(sender.calls(), vec!["answer_callback_query"; 2]);
|
||||||
}
|
}
|
||||||
respond(())
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,10 @@
|
|||||||
//! Bot command parsing, the `/`-command executor and `setMyCommands`
|
//! Bot command parsing, the `/`-command executor and `setMyCommands`
|
||||||
//! registration. URL/inline/callback flows live in their own modules.
|
//! registration. URL/inline/callback flows live in their own modules.
|
||||||
|
|
||||||
|
use super::urls::{PostSend, url_media};
|
||||||
use super::{CHAT_STORE, CONFIG, LINK_CACHE, log_key, reply, reply_html};
|
use super::{CHAT_STORE, CONFIG, LINK_CACHE, log_key, reply, reply_html};
|
||||||
|
use crate::ctx::AppContext;
|
||||||
|
use crate::state::ChatData;
|
||||||
use teloxide::RequestError;
|
use teloxide::RequestError;
|
||||||
use teloxide::prelude::*;
|
use teloxide::prelude::*;
|
||||||
use teloxide::types::{ChatId, Message, Recipient};
|
use teloxide::types::{ChatId, Message, Recipient};
|
||||||
@@ -31,30 +34,154 @@ pub(crate) enum Command {
|
|||||||
parse_with = "split"
|
parse_with = "split"
|
||||||
)]
|
)]
|
||||||
SetTemplate(String),
|
SetTemplate(String),
|
||||||
#[command(description = "Show chat state (debug)")]
|
#[command(description = "Remove a saved template", parse_with = "split")]
|
||||||
|
RemoveTemplate(String),
|
||||||
|
#[command(description = "Show this chat's settings")]
|
||||||
|
Settings,
|
||||||
|
#[command(description = "Show chat state (debug; admin only)")]
|
||||||
BotDict,
|
BotDict,
|
||||||
#[command(description = "Set site caption format", parse_with = "split")]
|
#[command(
|
||||||
|
description = "Set site caption format (- to reset)",
|
||||||
|
parse_with = parse_arg_remainder
|
||||||
|
)]
|
||||||
SetFormat(String),
|
SetFormat(String),
|
||||||
#[command(
|
#[command(
|
||||||
description = "Clear link cache (admin; optional URL, else all)",
|
description = "Clear link cache (admin; optional URL, else all)",
|
||||||
parse_with = "split"
|
parse_with = parse_arg_remainder
|
||||||
)]
|
)]
|
||||||
ClearCache(String),
|
ClearCache(String),
|
||||||
#[command(
|
#[command(
|
||||||
description = "Test link parsing (debug; no media sent)",
|
description = "Send a link's media (no forwarding)",
|
||||||
parse_with = parse_test_arg
|
parse_with = parse_arg_remainder
|
||||||
)]
|
)]
|
||||||
Test(String),
|
Test(String),
|
||||||
|
#[command(
|
||||||
|
description = "Parse a link and report it (debug; nothing sent)",
|
||||||
|
parse_with = parse_arg_remainder
|
||||||
|
)]
|
||||||
|
Debug(String),
|
||||||
}
|
}
|
||||||
|
|
||||||
/// `/test` argument parser: the whole remainder after the command name,
|
/// `/test` and `/debug` argument parser: the whole remainder after the command
|
||||||
/// trimmed. The built-in `split` parser takes exactly one space-separated
|
/// name, trimmed. The built-in `split` parser takes exactly one space-separated
|
||||||
/// token and rejects the rest, so a URL followed by a trailing space (or
|
/// token and rejects the rest, so a URL followed by a trailing space (or
|
||||||
/// pasted text) would silently fall through to the URL flow instead.
|
/// pasted text) would silently fall through to the URL flow instead.
|
||||||
fn parse_test_arg(s: String) -> Result<(String,), ParseError> {
|
fn parse_arg_remainder(s: String) -> Result<(String,), ParseError> {
|
||||||
Ok((s.trim().to_string(),))
|
Ok((s.trim().to_string(),))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Placeholders `/set_format` accepts, mirroring what
|
||||||
|
/// `x_media::site::caption_from_fields` substitutes.
|
||||||
|
const FORMAT_PLACEHOLDERS: [&str; 6] = ["url", "author", "author_url", "title", "content", "tags"];
|
||||||
|
|
||||||
|
/// `/start`'s welcome: what the bot is for, where links work, where to look
|
||||||
|
/// next. The old "Hello!" left a first-time user with nothing.
|
||||||
|
const START_TEXT: &str = "\
|
||||||
|
Send me a post link and I'll send back its images, videos and GIFs with the title, author and tags.
|
||||||
|
|
||||||
|
Supported: X/Twitter, Pixiv, Bluesky, Misskey (misskey.io), Bilibili.
|
||||||
|
In a private chat just paste the link. In a group, use inline mode (type @, pick me, then the link).
|
||||||
|
|
||||||
|
/help lists every command.";
|
||||||
|
|
||||||
|
/// Appended to `/help`'s command list: argument syntax, caption
|
||||||
|
/// placeholders and the private-chat rule — none of which teloxide's
|
||||||
|
/// `descriptions()` renders (it prints `/command — description` only).
|
||||||
|
const HELP_FOOTER: &str = "\
|
||||||
|
Arguments
|
||||||
|
/set_forward_channel <@channel or channel id>
|
||||||
|
/set_template <name> — reply to a message containing [] to save it
|
||||||
|
/remove_template <name> — see /settings for the saved names
|
||||||
|
/set_format <site> <format> — '-' restores the built-in format
|
||||||
|
/test <link> / /debug <link>
|
||||||
|
|
||||||
|
Caption placeholders (for /set_format)
|
||||||
|
{url} {author} {author_url} {title} {content} {tags}
|
||||||
|
A template's [] is replaced by the post link when forwarding.
|
||||||
|
|
||||||
|
Links are handled in private chats only; in a group use inline mode.";
|
||||||
|
|
||||||
|
/// Cap on template names echoed by `/settings`: a chat with hundreds of
|
||||||
|
/// templates must not produce a message Telegram rejects for length.
|
||||||
|
const MAX_SETTINGS_TEMPLATE_NAMES: usize = 30;
|
||||||
|
|
||||||
|
/// Sorted template names: the order `/settings`, `/remove_template` and the
|
||||||
|
/// prompt's buttons all show.
|
||||||
|
fn sorted_template_names(data: &ChatData) -> Vec<String> {
|
||||||
|
let mut names: Vec<String> = data.template.keys().cloned().collect();
|
||||||
|
names.sort();
|
||||||
|
names
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `/settings`: what this chat is configured to do, readable by anyone in it
|
||||||
|
/// (unlike `/bot_dict`, which dumps the raw state and is admin-only).
|
||||||
|
fn settings_text(data: &ChatData) -> String {
|
||||||
|
let mut lines = Vec::new();
|
||||||
|
match data.forward_channel_id {
|
||||||
|
Some(id) => lines.push(format!("Forward channel: {id}")),
|
||||||
|
None => lines.push(
|
||||||
|
"Forward channel: not set (use /set_forward_channel <@channel or id>)".to_string(),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
lines.push(format!(
|
||||||
|
"Edit before forward: {}",
|
||||||
|
if data.edit_before_forward {
|
||||||
|
"on"
|
||||||
|
} else {
|
||||||
|
"off"
|
||||||
|
}
|
||||||
|
));
|
||||||
|
let mut formats: Vec<String> = data
|
||||||
|
.message_format
|
||||||
|
.iter()
|
||||||
|
.map(|(site, format)| format!("{site} => {format}"))
|
||||||
|
.collect();
|
||||||
|
formats.sort();
|
||||||
|
lines.push(if formats.is_empty() {
|
||||||
|
"Caption formats: built-in for every site".to_string()
|
||||||
|
} else {
|
||||||
|
format!("Caption formats:\n {}", formats.join("\n "))
|
||||||
|
});
|
||||||
|
let names = sorted_template_names(data);
|
||||||
|
lines.push(match names.len() {
|
||||||
|
0 => "Templates: none".to_string(),
|
||||||
|
n => format!(
|
||||||
|
"Templates ({n}): {}{}",
|
||||||
|
names
|
||||||
|
.iter()
|
||||||
|
.take(MAX_SETTINGS_TEMPLATE_NAMES)
|
||||||
|
.cloned()
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(", "),
|
||||||
|
if n > MAX_SETTINGS_TEMPLATE_NAMES {
|
||||||
|
format!(", +{} more", n - MAX_SETTINGS_TEMPLATE_NAMES)
|
||||||
|
} else {
|
||||||
|
String::new()
|
||||||
|
}
|
||||||
|
),
|
||||||
|
});
|
||||||
|
lines.join("\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The first `{…}` token in a caption format that is not a known placeholder
|
||||||
|
/// (`None` when all of them are). The renderer replaces exact keys only, so an
|
||||||
|
/// unknown token would be published verbatim in every caption of that site —
|
||||||
|
/// caught here instead.
|
||||||
|
fn unknown_placeholder(format: &str) -> Option<&str> {
|
||||||
|
let mut rest = format;
|
||||||
|
while let Some(open) = rest.find('{') {
|
||||||
|
let after = &rest[open + 1..];
|
||||||
|
// An unclosed `{` is not a placeholder token at all.
|
||||||
|
let close = after.find('}')?;
|
||||||
|
let token = &after[..close];
|
||||||
|
if !FORMAT_PLACEHOLDERS.contains(&token) {
|
||||||
|
return Some(token);
|
||||||
|
}
|
||||||
|
rest = &after[close + 1..];
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
enum SetForwardChannelError {
|
enum SetForwardChannelError {
|
||||||
EmptyParameter,
|
EmptyParameter,
|
||||||
NotChannel,
|
NotChannel,
|
||||||
@@ -133,11 +260,17 @@ pub(crate) async fn execute_command(
|
|||||||
) -> Result<(), RequestError> {
|
) -> Result<(), RequestError> {
|
||||||
match command {
|
match command {
|
||||||
Command::Start => {
|
Command::Start => {
|
||||||
bot.send_message(message.chat.id, "Hello!").await?;
|
bot.send_message(message.chat.id, START_TEXT).await?;
|
||||||
}
|
}
|
||||||
Command::Help => {
|
Command::Help => {
|
||||||
bot.send_message(message.chat.id, Command::descriptions().to_string())
|
// The command list plus the parts teloxide's `descriptions()`
|
||||||
.await?;
|
// cannot show: argument syntax, caption placeholders, and where a
|
||||||
|
// link actually works.
|
||||||
|
bot.send_message(
|
||||||
|
message.chat.id,
|
||||||
|
format!("{}\n\n{}", Command::descriptions(), HELP_FOOTER),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
}
|
}
|
||||||
Command::SetForwardChannel(channel) => {
|
Command::SetForwardChannel(channel) => {
|
||||||
let result = match set_forward_channel_handler(bot, message, channel).await {
|
let result = match set_forward_channel_handler(bot, message, channel).await {
|
||||||
@@ -225,10 +358,64 @@ pub(crate) async fn execute_command(
|
|||||||
};
|
};
|
||||||
reply(bot, message.chat.id.0, message.id, text).await?;
|
reply(bot, message.chat.id.0, message.id, text).await?;
|
||||||
}
|
}
|
||||||
|
Command::RemoveTemplate(name) => {
|
||||||
|
let chat_id = message.chat.id.0;
|
||||||
|
let name = name.trim().to_string();
|
||||||
|
if name.is_empty() {
|
||||||
|
reply(
|
||||||
|
bot,
|
||||||
|
chat_id,
|
||||||
|
message.id,
|
||||||
|
"Usage: /remove_template <name> (see /settings for the saved names)",
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let removed = CHAT_STORE
|
||||||
|
.update(chat_id, |data| data.template.remove(&name).is_some())
|
||||||
|
.await;
|
||||||
|
let text = if removed {
|
||||||
|
format!("Template '{name}' removed.")
|
||||||
|
} else {
|
||||||
|
// Name the live templates: a typo would otherwise look like a
|
||||||
|
// successful delete.
|
||||||
|
let names = sorted_template_names(&CHAT_STORE.get(chat_id).await);
|
||||||
|
if names.is_empty() {
|
||||||
|
format!("No template named '{name}'. None are saved yet.")
|
||||||
|
} else {
|
||||||
|
format!("No template named '{name}'. Saved: {}", names.join(", "))
|
||||||
|
}
|
||||||
|
};
|
||||||
|
reply(bot, chat_id, message.id, text).await?;
|
||||||
|
}
|
||||||
|
Command::Settings => {
|
||||||
|
let chat_id = message.chat.id.0;
|
||||||
|
let data = CHAT_STORE.get(chat_id).await;
|
||||||
|
reply(bot, chat_id, message.id, settings_text(&data)).await?;
|
||||||
|
}
|
||||||
Command::BotDict => {
|
Command::BotDict => {
|
||||||
|
// Debug dump of the chat's persisted state: admin only (it echoes
|
||||||
|
// forward-channel ids and templates to whoever asks).
|
||||||
|
let sender_id = message
|
||||||
|
.from
|
||||||
|
.as_ref()
|
||||||
|
.map(|user| user.id.0 as i64)
|
||||||
|
.unwrap_or(-1);
|
||||||
|
if !CONFIG.admin_ids.contains(&sender_id) {
|
||||||
|
reply(bot, message.chat.id.0, message.id, "Admin only.").await?;
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
let chat_data = CHAT_STORE.get(message.chat.id.0).await;
|
let chat_data = CHAT_STORE.get(message.chat.id.0).await;
|
||||||
let debug = format!("{chat_data:?}");
|
let debug = html_escape::encode_text(&format!("{chat_data:?}")).into_owned();
|
||||||
let text = html_escape::encode_text(&debug).into_owned();
|
// A chat with many templates/edit records exceeds Telegram's 4096
|
||||||
|
// char message limit; the dump is plain text (no parse mode), so a
|
||||||
|
// plain byte-boundary cut is safe.
|
||||||
|
let end = debug.floor_char_boundary(MAX_DEBUG_DUMP_CHARS.min(debug.len()));
|
||||||
|
let text = if end < debug.len() {
|
||||||
|
format!("{}…", &debug[..end])
|
||||||
|
} else {
|
||||||
|
debug
|
||||||
|
};
|
||||||
reply(bot, message.chat.id.0, message.id, text).await?;
|
reply(bot, message.chat.id.0, message.id, text).await?;
|
||||||
}
|
}
|
||||||
Command::SetFormat(arg) => {
|
Command::SetFormat(arg) => {
|
||||||
@@ -253,7 +440,45 @@ pub(crate) async fn execute_command(
|
|||||||
bot,
|
bot,
|
||||||
message.chat.id.0,
|
message.chat.id.0,
|
||||||
message.id,
|
message.id,
|
||||||
"Unknown site. Use twitter, bsky, pixiv or misskey.",
|
"Unknown site. Use twitter, bsky, pixiv, misskey or bilibili.",
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
// `-` resets to the site's built-in caption: without it a chat that
|
||||||
|
// set a format once could never get back to the default (the
|
||||||
|
// built-in format string is not something a user can retype).
|
||||||
|
if format == "-" {
|
||||||
|
CHAT_STORE
|
||||||
|
.update(chat_id, |data| {
|
||||||
|
data.message_format.remove(site);
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
reply(
|
||||||
|
bot,
|
||||||
|
message.chat.id.0,
|
||||||
|
message.id,
|
||||||
|
"Format reset to the built-in one.",
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
// A typo like {titel} would otherwise be rendered literally into
|
||||||
|
// every caption of that site (the renderer only substitutes the
|
||||||
|
// exact keys), which is invisible until a post arrives.
|
||||||
|
if let Some(token) = unknown_placeholder(&format) {
|
||||||
|
reply(
|
||||||
|
bot,
|
||||||
|
message.chat.id.0,
|
||||||
|
message.id,
|
||||||
|
format!(
|
||||||
|
"Unknown placeholder {{{token}}}. Available: {}",
|
||||||
|
FORMAT_PLACEHOLDERS
|
||||||
|
.iter()
|
||||||
|
.map(|name| format!("{{{name}}}"))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(" ")
|
||||||
|
),
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
return Ok(());
|
return Ok(());
|
||||||
@@ -263,7 +488,13 @@ pub(crate) async fn execute_command(
|
|||||||
data.message_format.insert(site.to_string(), format);
|
data.message_format.insert(site.to_string(), format);
|
||||||
})
|
})
|
||||||
.await;
|
.await;
|
||||||
reply(bot, message.chat.id.0, message.id, "Format set.").await?;
|
reply(
|
||||||
|
bot,
|
||||||
|
message.chat.id.0,
|
||||||
|
message.id,
|
||||||
|
"Format set. Use /debug <link> to preview the caption.",
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
}
|
}
|
||||||
Command::ClearCache(arg) => {
|
Command::ClearCache(arg) => {
|
||||||
let sender_id = message
|
let sender_id = message
|
||||||
@@ -294,7 +525,7 @@ pub(crate) async fn execute_command(
|
|||||||
bot,
|
bot,
|
||||||
message.chat.id.0,
|
message.chat.id.0,
|
||||||
message.id,
|
message.id,
|
||||||
"Unrecognized link. Use a twitter/x, pixiv, bsky or misskey post URL.",
|
"Unrecognized link. Use a twitter/x, pixiv, bsky, misskey or bilibili post URL.",
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
return Ok(());
|
return Ok(());
|
||||||
@@ -327,17 +558,54 @@ pub(crate) async fn execute_command(
|
|||||||
.await?;
|
.await?;
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
if x_media::site::cache_key(url).is_none() {
|
||||||
|
reply(
|
||||||
|
bot,
|
||||||
|
message.chat.id.0,
|
||||||
|
message.id,
|
||||||
|
"No enabled site matches this link (twitter/x, pixiv, bsky, misskey or bilibili).",
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
// The ordinary link pipeline with the chat's post-send actions
|
||||||
|
// suppressed: the media is sent (and cached) like a normal link,
|
||||||
|
// but nothing is forwarded to the channel and no
|
||||||
|
// edit-before-forward prompt opens. Info level echoes the
|
||||||
|
// normalized key (never the raw URL) per the logging convention.
|
||||||
|
log::info!("test: sending [key={}]", log_key(url));
|
||||||
|
let ctx = AppContext::from_statics(bot);
|
||||||
|
url_media(
|
||||||
|
&ctx,
|
||||||
|
message.chat.id.0,
|
||||||
|
message.id.0 as i64,
|
||||||
|
url,
|
||||||
|
PostSend::Suppressed,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
Command::Debug(arg) => {
|
||||||
|
let url = arg.trim();
|
||||||
|
if url.is_empty() {
|
||||||
|
reply(
|
||||||
|
bot,
|
||||||
|
message.chat.id.0,
|
||||||
|
message.id,
|
||||||
|
"Usage: /debug <post url>",
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
// Debug tool: report the parse result only — nothing is sent,
|
// Debug tool: report the parse result only — nothing is sent,
|
||||||
// cached or forwarded. Info level echoes the normalized key
|
// cached or forwarded.
|
||||||
// (never the raw URL) per the logging convention.
|
log::info!("debug: parsing [key={}]", log_key(url));
|
||||||
log::info!("test: parsing [key={}]", log_key(url));
|
|
||||||
match x_media::site::fetch(url).await {
|
match x_media::site::fetch(url).await {
|
||||||
Ok(None) => {
|
Ok(None) => {
|
||||||
reply(
|
reply(
|
||||||
bot,
|
bot,
|
||||||
message.chat.id.0,
|
message.chat.id.0,
|
||||||
message.id,
|
message.id,
|
||||||
"No enabled site matches this link (twitter/x, pixiv, bsky or misskey).",
|
"No enabled site matches this link (twitter/x, pixiv, bsky, misskey or bilibili).",
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
}
|
}
|
||||||
@@ -351,14 +619,33 @@ pub(crate) async fn execute_command(
|
|||||||
.await?;
|
.await?;
|
||||||
}
|
}
|
||||||
Ok(Some(fetched)) => {
|
Ok(Some(fetched)) => {
|
||||||
let report = test_parse_report(
|
// The preview must show what a link would actually send:
|
||||||
|
// the chat's per-site format override plus the long-post
|
||||||
|
// quoting. Rendering the raw built-in caption here made
|
||||||
|
// `/set_format` look like it did nothing.
|
||||||
|
let format = CHAT_STORE
|
||||||
|
.get(message.chat.id.0)
|
||||||
|
.await
|
||||||
|
.message_format
|
||||||
|
.get(fetched.site_name())
|
||||||
|
.cloned()
|
||||||
|
.unwrap_or_default();
|
||||||
|
let caption = preview_caption(
|
||||||
|
&format,
|
||||||
|
&fetched.caption,
|
||||||
|
&fetched.source_url,
|
||||||
|
fetched.render_fields(),
|
||||||
|
CONFIG.caption_quote_text_chars,
|
||||||
|
);
|
||||||
|
let report = debug_report(
|
||||||
url,
|
url,
|
||||||
fetched.site_name(),
|
fetched.site_name(),
|
||||||
&fetched.source_url,
|
&fetched.source_url,
|
||||||
&fetched.title,
|
&fetched.title,
|
||||||
|
&fetched.content,
|
||||||
fetched.render_fields(),
|
fetched.render_fields(),
|
||||||
fetched.sensitive,
|
fetched.sensitive,
|
||||||
&fetched.caption,
|
&caption,
|
||||||
&fetched.media,
|
&fetched.media,
|
||||||
);
|
);
|
||||||
// HTML report: the caption renders inside a <blockquote>
|
// HTML report: the caption renders inside a <blockquote>
|
||||||
@@ -376,20 +663,69 @@ fn plural(n: usize) -> &'static str {
|
|||||||
if n == 1 { "y" } else { "ies" }
|
if n == 1 { "y" } else { "ies" }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Bot profile texts (Bot API `setMyDescription` / `setMyShortDescription`):
|
||||||
|
/// shown on the bot's profile page and in the share sheet. Without them a
|
||||||
|
/// shared link says nothing about what the bot does.
|
||||||
|
const BOT_DESCRIPTION: &str = "\
|
||||||
|
Send a post link from X/Twitter, Pixiv, Bluesky, Misskey (misskey.io) or Bilibili and get its images, videos and GIFs back with the title, author and tags.
|
||||||
|
Links are handled in private chats; a group can use inline mode. /help lists every command.";
|
||||||
|
const BOT_SHORT_DESCRIPTION: &str =
|
||||||
|
"Post links (X, Pixiv, Bluesky, Misskey, Bilibili) -> media messages";
|
||||||
|
|
||||||
/// Registers the bot's command list with Telegram so clients show it in the
|
/// Registers the bot's command list with Telegram so clients show it in the
|
||||||
/// `/` menu (Bot API `setMyCommands`).
|
/// `/` menu (Bot API `setMyCommands`), plus its profile description texts.
|
||||||
pub async fn register_commands(bot: &Bot) -> Result<(), RequestError> {
|
pub async fn register_commands(bot: &Bot) -> Result<(), RequestError> {
|
||||||
let commands = Command::bot_commands();
|
let commands = Command::bot_commands();
|
||||||
bot.set_my_commands(commands.clone()).await?;
|
bot.set_my_commands(commands.clone()).await?;
|
||||||
log::info!("registered {} commands", commands.len());
|
log::info!("registered {} commands", commands.len());
|
||||||
|
// Profile texts are cosmetic: a failure (rare) must not abort startup.
|
||||||
|
if let Err(e) = bot.set_my_description().description(BOT_DESCRIPTION).await {
|
||||||
|
log::warn!("failed to set the bot description: {e}");
|
||||||
|
}
|
||||||
|
if let Err(e) = bot
|
||||||
|
.set_my_short_description()
|
||||||
|
.short_description(BOT_SHORT_DESCRIPTION)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
log::warn!("failed to set the bot short description: {e}");
|
||||||
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Telegram's plain-text message limit is 4096 chars; the report stays under
|
/// Telegram's plain-text message limit is 4096 chars; the report stays under
|
||||||
/// it even for very large threads (many media lines + a long caption).
|
/// it even for very large threads (many media lines + a long caption).
|
||||||
const MAX_TEST_REPORT_CHARS: usize = 4000;
|
const MAX_DEBUG_REPORT_CHARS: usize = 4000;
|
||||||
|
|
||||||
/// Builds the HTML report for the `/test` command: what the parser produced
|
/// Cap for the `/bot_dict` debug dump: the state is echoed as one plain-text
|
||||||
|
/// message, so it must stay under Telegram's 4096-char limit.
|
||||||
|
const MAX_DEBUG_DUMP_CHARS: usize = 3500;
|
||||||
|
|
||||||
|
/// The caption a link would actually send for this chat: the per-site format
|
||||||
|
/// override (empty = the site's built-in caption) and, on a long post, the
|
||||||
|
/// same text quoting the send paths apply. `/debug` shows this so the preview
|
||||||
|
/// cannot drift from what the send paths produce.
|
||||||
|
fn preview_caption(
|
||||||
|
format: &str,
|
||||||
|
built_in: &str,
|
||||||
|
url: &str,
|
||||||
|
fields: Option<(&str, &str, &str, &str, &str)>,
|
||||||
|
quote_chars: usize,
|
||||||
|
) -> String {
|
||||||
|
let caption = match fields {
|
||||||
|
// Same call the send paths make through `Fetched::caption_with`: an
|
||||||
|
// empty format falls back to the built-in caption.
|
||||||
|
Some((author, author_url, title, content, tags)) => x_media::site::caption_from_fields(
|
||||||
|
format, built_in, url, author, author_url, title, content, tags,
|
||||||
|
),
|
||||||
|
None => x_media::site::truncate_caption(built_in),
|
||||||
|
};
|
||||||
|
let text = fields
|
||||||
|
.map(|(_, _, title, content, _)| x_media::site::compose_text(title, content))
|
||||||
|
.unwrap_or_default();
|
||||||
|
crate::send::quote_long_caption(&caption, &text, quote_chars).into_owned()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Builds the HTML report for the `/debug` command: what the parser produced
|
||||||
/// for a link (site, canonical URL, title/author/tags, caption and the media
|
/// for a link (site, canonical URL, title/author/tags, caption and the media
|
||||||
/// list) — no media is sent and nothing is cached or forwarded. Sent with
|
/// list) — no media is sent and nothing is cached or forwarded. Sent with
|
||||||
/// HTML parse mode: raw fields are escaped, the pre-escaped render fields are
|
/// HTML parse mode: raw fields are escaped, the pre-escaped render fields are
|
||||||
@@ -399,12 +735,13 @@ const MAX_TEST_REPORT_CHARS: usize = 4000;
|
|||||||
/// constructing a `Fetched` (its render fields are `pub(crate)` to the
|
/// constructing a `Fetched` (its render fields are `pub(crate)` to the
|
||||||
/// x-media crate).
|
/// x-media crate).
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
fn test_parse_report(
|
fn debug_report(
|
||||||
url: &str,
|
url: &str,
|
||||||
site_id: &str,
|
site_id: &str,
|
||||||
source_url: &str,
|
source_url: &str,
|
||||||
title: &str,
|
title: &str,
|
||||||
render: Option<(&str, &str, &str, &str)>,
|
content: &str,
|
||||||
|
render: Option<(&str, &str, &str, &str, &str)>,
|
||||||
sensitive: bool,
|
sensitive: bool,
|
||||||
caption: &str,
|
caption: &str,
|
||||||
media: &[x_media::media::Media],
|
media: &[x_media::media::Media],
|
||||||
@@ -424,7 +761,8 @@ fn test_parse_report(
|
|||||||
html_escape::encode_text(source_url)
|
html_escape::encode_text(source_url)
|
||||||
));
|
));
|
||||||
lines.push(format!("title: {}", html_escape::encode_text(title)));
|
lines.push(format!("title: {}", html_escape::encode_text(title)));
|
||||||
if let Some((author, author_url, _title, tags)) = render {
|
lines.push(format!("content: {}", html_escape::encode_text(content)));
|
||||||
|
if let Some((author, author_url, _title, _content, tags)) = render {
|
||||||
// The render fields are already pre-escaped for HTML captions; embed
|
// The render fields are already pre-escaped for HTML captions; embed
|
||||||
// them as-is so the report renders them exactly like the final
|
// them as-is so the report renders them exactly like the final
|
||||||
// caption. `author_url` is raw and gets escaped here.
|
// caption. `author_url` is raw and gets escaped here.
|
||||||
@@ -460,8 +798,8 @@ fn test_parse_report(
|
|||||||
"
|
"
|
||||||
",
|
",
|
||||||
);
|
);
|
||||||
if out.chars().count() > MAX_TEST_REPORT_CHARS {
|
if out.chars().count() > MAX_DEBUG_REPORT_CHARS {
|
||||||
let end = out.floor_char_boundary(MAX_TEST_REPORT_CHARS - 1);
|
let end = out.floor_char_boundary(MAX_DEBUG_REPORT_CHARS - 1);
|
||||||
out = format!("{}…", &out[..end]);
|
out = format!("{}…", &out[..end]);
|
||||||
}
|
}
|
||||||
out
|
out
|
||||||
@@ -469,11 +807,13 @@ fn test_parse_report(
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{MAX_TEST_REPORT_CHARS, test_parse_report};
|
use super::{
|
||||||
|
MAX_DEBUG_REPORT_CHARS, debug_report, preview_caption, settings_text, unknown_placeholder,
|
||||||
|
};
|
||||||
use x_media::media::Media;
|
use x_media::media::Media;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_parse_report_renders_fields_and_media() {
|
fn debug_report_renders_fields_and_media() {
|
||||||
let media = vec![
|
let media = vec![
|
||||||
Media::Illustration {
|
Media::Illustration {
|
||||||
title: None,
|
title: None,
|
||||||
@@ -487,12 +827,19 @@ mod tests {
|
|||||||
thumbnail_url: "https://cdn.example/2.jpg".into(),
|
thumbnail_url: "https://cdn.example/2.jpg".into(),
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
let report = test_parse_report(
|
let report = debug_report(
|
||||||
"https://x.com/u/status/1",
|
"https://x.com/u/status/1",
|
||||||
"twitter",
|
"twitter",
|
||||||
"https://x.com/u/status/1",
|
"https://x.com/u/status/1",
|
||||||
"My title",
|
"My title",
|
||||||
Some(("Author", "https://x.com/u", "My title", "tag1 tag2")),
|
"My content",
|
||||||
|
Some((
|
||||||
|
"Author",
|
||||||
|
"https://x.com/u",
|
||||||
|
"My title",
|
||||||
|
"My content",
|
||||||
|
"tag1 tag2",
|
||||||
|
)),
|
||||||
false,
|
false,
|
||||||
"<a href=\"https://x.com/u\">Author</a> · My title",
|
"<a href=\"https://x.com/u\">Author</a> · My title",
|
||||||
&media,
|
&media,
|
||||||
@@ -500,6 +847,7 @@ mod tests {
|
|||||||
assert!(report.contains("site: twitter"), "{report}");
|
assert!(report.contains("site: twitter"), "{report}");
|
||||||
assert!(report.contains("key: twitter:1"), "{report}");
|
assert!(report.contains("key: twitter:1"), "{report}");
|
||||||
assert!(report.contains("title: My title"), "{report}");
|
assert!(report.contains("title: My title"), "{report}");
|
||||||
|
assert!(report.contains("content: My content"), "{report}");
|
||||||
assert!(report.contains("author: Author"), "{report}");
|
assert!(report.contains("author: Author"), "{report}");
|
||||||
assert!(report.contains("author_url: https://x.com/u"), "{report}");
|
assert!(report.contains("author_url: https://x.com/u"), "{report}");
|
||||||
assert!(report.contains("tags: tag1 tag2"), "{report}");
|
assert!(report.contains("tags: tag1 tag2"), "{report}");
|
||||||
@@ -516,28 +864,30 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_parse_report_without_render_data_and_no_media() {
|
fn debug_report_without_render_data_and_no_media() {
|
||||||
let report = test_parse_report("u", "pixiv", "s", "t", None, true, "c", &[]);
|
let report = debug_report("u", "pixiv", "s", "t", "c", None, true, "p", &[]);
|
||||||
assert!(!report.contains("author:"), "{report}");
|
assert!(!report.contains("author:"), "{report}");
|
||||||
assert!(report.contains("sensitive: true"), "{report}");
|
assert!(report.contains("sensitive: true"), "{report}");
|
||||||
assert!(report.contains("media (0):"), "{report}");
|
assert!(report.contains("media (0):"), "{report}");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_parse_report_wraps_caption_in_blockquote() {
|
fn debug_report_wraps_caption_in_blockquote() {
|
||||||
// The report is an HTML message: raw fields are escaped, pre-escaped
|
// The report is an HTML message: raw fields are escaped, pre-escaped
|
||||||
// render fields are embedded as-is, and the caption is wrapped in a
|
// render fields are embedded as-is, and the caption is wrapped in a
|
||||||
// <blockquote> so it shows exactly as it will render in the sent
|
// <blockquote> so it shows exactly as it will render in the sent
|
||||||
// media caption (escaped text and links included).
|
// media caption (escaped text and links included).
|
||||||
let report = test_parse_report(
|
let report = debug_report(
|
||||||
"https://x.com/u/status/1",
|
"https://x.com/u/status/1",
|
||||||
"twitter",
|
"twitter",
|
||||||
"https://x.com/u/status/1",
|
"https://x.com/u/status/1",
|
||||||
"A & B <C>",
|
"A & B <C>",
|
||||||
|
"body & <more>",
|
||||||
Some((
|
Some((
|
||||||
"A & B",
|
"A & B",
|
||||||
"https://x.com/u",
|
"https://x.com/u",
|
||||||
"A & B <C>",
|
"A & B <C>",
|
||||||
|
"body & <more>",
|
||||||
"#a & #b",
|
"#a & #b",
|
||||||
)),
|
)),
|
||||||
false,
|
false,
|
||||||
@@ -563,7 +913,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_parse_report_is_capped() {
|
fn debug_report_is_capped() {
|
||||||
// 200 media lines ≈ 8 KB, comfortably over the cap.
|
// 200 media lines ≈ 8 KB, comfortably over the cap.
|
||||||
let media: Vec<Media> = (0..200)
|
let media: Vec<Media> = (0..200)
|
||||||
.map(|i| Media::Illustration {
|
.map(|i| Media::Illustration {
|
||||||
@@ -573,8 +923,265 @@ mod tests {
|
|||||||
fallback_url: None,
|
fallback_url: None,
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
let report = test_parse_report("u", "twitter", "s", "t", None, false, "c", &media);
|
let report = debug_report("u", "twitter", "s", "t", "c", None, false, "p", &media);
|
||||||
assert!(report.chars().count() <= MAX_TEST_REPORT_CHARS, "{report}");
|
assert!(report.chars().count() <= MAX_DEBUG_REPORT_CHARS, "{report}");
|
||||||
assert!(report.ends_with('…'), "{report}");
|
assert!(report.ends_with('…'), "{report}");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn settings_text_reports_the_chat_configuration() {
|
||||||
|
use crate::state::ChatData;
|
||||||
|
|
||||||
|
// A fresh chat: the defaults must be spelled out, including how to set
|
||||||
|
// the channel (an empty field is not a status).
|
||||||
|
let empty = settings_text(&ChatData::default());
|
||||||
|
assert!(empty.contains("Forward channel: not set"), "{empty}");
|
||||||
|
assert!(empty.contains("/set_forward_channel"), "{empty}");
|
||||||
|
assert!(empty.contains("Edit before forward: off"), "{empty}");
|
||||||
|
assert!(empty.contains("built-in for every site"), "{empty}");
|
||||||
|
assert!(empty.contains("Templates: none"), "{empty}");
|
||||||
|
|
||||||
|
let configured = ChatData {
|
||||||
|
forward_channel_id: Some(-100123),
|
||||||
|
edit_before_forward: true,
|
||||||
|
template: [("b", "[]"), ("a", "[]")]
|
||||||
|
.into_iter()
|
||||||
|
.map(|(k, v)| (k.to_string(), v.to_string()))
|
||||||
|
.collect(),
|
||||||
|
message_format: [("twitter", "{author}: {content}")]
|
||||||
|
.into_iter()
|
||||||
|
.map(|(k, v)| (k.to_string(), v.to_string()))
|
||||||
|
.collect(),
|
||||||
|
..ChatData::default()
|
||||||
|
};
|
||||||
|
let text = settings_text(&configured);
|
||||||
|
assert!(text.contains("Forward channel: -100123"), "{text}");
|
||||||
|
assert!(text.contains("Edit before forward: on"), "{text}");
|
||||||
|
assert!(text.contains("twitter => {author}: {content}"), "{text}");
|
||||||
|
// Sorted, so the same chat always reports the same thing.
|
||||||
|
assert!(text.contains("Templates (2): a, b"), "{text}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn help_and_start_cover_what_the_command_list_cannot() {
|
||||||
|
// The placeholders the renderer substitutes must be the ones the help
|
||||||
|
// lists: a stale list is worse than none.
|
||||||
|
for placeholder in super::FORMAT_PLACEHOLDERS {
|
||||||
|
assert!(
|
||||||
|
super::HELP_FOOTER.contains(&format!("{{{placeholder}}}")),
|
||||||
|
"help does not document {{{placeholder}}}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// The private-chat rule and the template placeholder semantics are the
|
||||||
|
// two things users got wrong most often.
|
||||||
|
assert!(super::HELP_FOOTER.contains("private chats only"));
|
||||||
|
assert!(super::HELP_FOOTER.contains("[]"));
|
||||||
|
assert!(super::START_TEXT.contains("inline mode"));
|
||||||
|
assert!(super::START_TEXT.contains("/help"));
|
||||||
|
// Both must stay inside Telegram's message limit.
|
||||||
|
assert!(super::HELP_FOOTER.chars().count() < 2000);
|
||||||
|
assert!(super::START_TEXT.chars().count() < 2000);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn every_command_is_registered_and_parses() {
|
||||||
|
use teloxide::utils::command::BotCommands;
|
||||||
|
|
||||||
|
use super::Command;
|
||||||
|
|
||||||
|
let registered: Vec<String> = Command::bot_commands()
|
||||||
|
.into_iter()
|
||||||
|
.map(|command| command.command.trim_start_matches('/').to_string())
|
||||||
|
.collect();
|
||||||
|
for expected in [
|
||||||
|
"start",
|
||||||
|
"help",
|
||||||
|
"settings",
|
||||||
|
"set_forward_channel",
|
||||||
|
"remove_template",
|
||||||
|
"set_format",
|
||||||
|
"test",
|
||||||
|
"debug",
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
registered.iter().any(|name| name == expected),
|
||||||
|
"{expected} missing from {registered:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// Telegram caps a command description at 256 chars.
|
||||||
|
for command in Command::bot_commands() {
|
||||||
|
assert!(
|
||||||
|
command.description.chars().count() <= 256,
|
||||||
|
"{}: description too long",
|
||||||
|
command.command
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// A command with a `String` argument must parse with its whole
|
||||||
|
// argument: without `parse_with`, teloxide's default parser rejects
|
||||||
|
// `/remove_template x` and the command silently falls through to the
|
||||||
|
// URL flow.
|
||||||
|
assert!(matches!(
|
||||||
|
Command::parse("/settings", ""),
|
||||||
|
Ok(Command::Settings)
|
||||||
|
));
|
||||||
|
match Command::parse("/remove_template tpl", "") {
|
||||||
|
Ok(Command::RemoveTemplate(name)) => assert_eq!(name, "tpl"),
|
||||||
|
Ok(_) => panic!("/remove_template parsed as another command"),
|
||||||
|
Err(e) => panic!("parse error: {e}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn every_documented_invocation_parses() {
|
||||||
|
use teloxide::utils::command::BotCommands;
|
||||||
|
|
||||||
|
use super::Command;
|
||||||
|
|
||||||
|
// The README's forms, verbatim. teloxide's `split` parser accepts
|
||||||
|
// EXACTLY one token per `String` field, so a command documented with
|
||||||
|
// two arguments (or an optional one) silently stops parsing — and a
|
||||||
|
// command that does not parse falls through to the URL flow in
|
||||||
|
// silence.
|
||||||
|
type Check = fn(&Command) -> bool;
|
||||||
|
let cases: Vec<(&str, Check)> = vec![
|
||||||
|
("/start", |c| matches!(c, Command::Start)),
|
||||||
|
("/help", |c| matches!(c, Command::Help)),
|
||||||
|
("/settings", |c| matches!(c, Command::Settings)),
|
||||||
|
("/edit_before_forward", |c| {
|
||||||
|
matches!(c, Command::EditBeforeForward)
|
||||||
|
}),
|
||||||
|
("/remove_forward_channel", |c| {
|
||||||
|
matches!(c, Command::RemoveForwardChannel)
|
||||||
|
}),
|
||||||
|
("/bot_dict", |c| matches!(c, Command::BotDict)),
|
||||||
|
(
|
||||||
|
"/set_forward_channel @a_channel",
|
||||||
|
|c| matches!(c, Command::SetForwardChannel(a) if a == "@a_channel"),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"/set_template tpl",
|
||||||
|
|c| matches!(c, Command::SetTemplate(a) if a == "tpl"),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"/remove_template tpl",
|
||||||
|
|c| matches!(c, Command::RemoveTemplate(a) if a == "tpl"),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"/set_format twitter {author}: {title}",
|
||||||
|
|c| matches!(c, Command::SetFormat(a) if a == "twitter {author}: {title}"),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"/set_format twitter -",
|
||||||
|
|c| matches!(c, Command::SetFormat(a) if a == "twitter -"),
|
||||||
|
),
|
||||||
|
// Documented as "clear everything" when called without a link.
|
||||||
|
(
|
||||||
|
"/clear_cache",
|
||||||
|
|c| matches!(c, Command::ClearCache(a) if a.is_empty()),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"/clear_cache https://x.com/u/status/1",
|
||||||
|
|c| matches!(c, Command::ClearCache(a) if a == "https://x.com/u/status/1"),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"/test https://x.com/u/status/1",
|
||||||
|
|c| matches!(c, Command::Test(a) if a == "https://x.com/u/status/1"),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"/debug https://x.com/u/status/1",
|
||||||
|
|c| matches!(c, Command::Debug(a) if a == "https://x.com/u/status/1"),
|
||||||
|
),
|
||||||
|
];
|
||||||
|
|
||||||
|
for (text, ok) in cases {
|
||||||
|
match Command::parse(text, "") {
|
||||||
|
Ok(parsed) => assert!(ok(&parsed), "{text} parsed as the wrong variant"),
|
||||||
|
Err(e) => panic!("{text} did not parse: {e}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn preview_caption_applies_the_chat_format_and_the_long_post_quote() {
|
||||||
|
let fields = Some((
|
||||||
|
"Author",
|
||||||
|
"https://x.com/u",
|
||||||
|
"Pinned title",
|
||||||
|
"Pinned body",
|
||||||
|
"#tag",
|
||||||
|
));
|
||||||
|
|
||||||
|
// No format override → the site's built-in caption, untouched.
|
||||||
|
assert_eq!(
|
||||||
|
preview_caption(
|
||||||
|
"",
|
||||||
|
"built-in caption",
|
||||||
|
"https://x.com/u/status/1",
|
||||||
|
fields,
|
||||||
|
200
|
||||||
|
),
|
||||||
|
"built-in caption"
|
||||||
|
);
|
||||||
|
|
||||||
|
// The bug this pins: `/debug` used to print the built-in caption even
|
||||||
|
// with a format set, so `/set_format` looked like it did nothing.
|
||||||
|
let formatted = preview_caption(
|
||||||
|
"{author} · {title}",
|
||||||
|
"built-in caption",
|
||||||
|
"https://x.com/u/status/1",
|
||||||
|
fields,
|
||||||
|
200,
|
||||||
|
);
|
||||||
|
assert_eq!(formatted, "Author · Pinned title");
|
||||||
|
|
||||||
|
// `{url}` comes from the canonical post URL, as in the send paths.
|
||||||
|
assert_eq!(
|
||||||
|
preview_caption(
|
||||||
|
"{url} {title}",
|
||||||
|
"built-in",
|
||||||
|
"https://x.com/u/status/1",
|
||||||
|
fields,
|
||||||
|
200
|
||||||
|
),
|
||||||
|
"https://x.com/u/status/1 Pinned title"
|
||||||
|
);
|
||||||
|
|
||||||
|
// A long post's text is quoted exactly like the send paths quote it.
|
||||||
|
let long = "正".repeat(300);
|
||||||
|
let fields = Some(("Author", "https://x.com/u", "", long.as_str(), ""));
|
||||||
|
let quoted = preview_caption(
|
||||||
|
"",
|
||||||
|
"https://x.com/u/status/1\n<a href=\"https://x.com/u\">Author</a>: 正…",
|
||||||
|
"https://x.com/u/status/1",
|
||||||
|
fields,
|
||||||
|
200,
|
||||||
|
);
|
||||||
|
assert!(quoted.contains("<blockquote expandable>"), "{quoted}");
|
||||||
|
|
||||||
|
// Without render fields (a site that does not expose them) the
|
||||||
|
// built-in caption is all there is.
|
||||||
|
assert_eq!(
|
||||||
|
preview_caption("", "built-in", "https://x.com/u/status/1", None, 200),
|
||||||
|
"built-in"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unknown_placeholder_finds_typos_only() {
|
||||||
|
assert_eq!(unknown_placeholder("{author} — {title}"), None);
|
||||||
|
// Every key the renderer substitutes must pass, in any combination.
|
||||||
|
assert_eq!(
|
||||||
|
unknown_placeholder("{url}{author}{author_url}{title}{content}{tags}"),
|
||||||
|
None
|
||||||
|
);
|
||||||
|
// Plain text and braces Telegram renders literally are not tokens.
|
||||||
|
assert_eq!(unknown_placeholder("no placeholders here"), None);
|
||||||
|
assert_eq!(unknown_placeholder("{unclosed"), None);
|
||||||
|
|
||||||
|
assert_eq!(unknown_placeholder("{titel}"), Some("titel"));
|
||||||
|
assert_eq!(unknown_placeholder("{title} {Content}"), Some("Content"));
|
||||||
|
// A typo after a valid token is still found.
|
||||||
|
assert_eq!(unknown_placeholder("{url} {tag}"), Some("tag"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
//! inline cache instead of re-fetching.
|
//! inline cache instead of re-fetching.
|
||||||
|
|
||||||
use super::log_key;
|
use super::log_key;
|
||||||
|
use std::collections::HashMap;
|
||||||
use std::sync::LazyLock;
|
use std::sync::LazyLock;
|
||||||
use teloxide::RequestError;
|
use teloxide::RequestError;
|
||||||
use teloxide::prelude::*;
|
use teloxide::prelude::*;
|
||||||
@@ -19,16 +20,65 @@ use x_media::media::Media;
|
|||||||
/// post id. Only answer once the query has been stable for this long.
|
/// post id. Only answer once the query has been stable for this long.
|
||||||
const INLINE_DEBOUNCE: std::time::Duration = std::time::Duration::from_millis(800);
|
const INLINE_DEBOUNCE: std::time::Duration = std::time::Duration::from_millis(800);
|
||||||
|
|
||||||
/// Last seen inline query and whether it was already answered. Guards the
|
/// Last seen inline query per user and whether it was already answered.
|
||||||
/// debounce timer: a repeat of an answered query is served by Telegram's
|
/// Guards the debounce timer: a repeat of an answered query is served by
|
||||||
/// inline cache (see `cache_time`), not by another fetch.
|
/// Telegram's inline cache (see `cache_time`), not by another fetch. Keyed by
|
||||||
|
/// user id — a single shared slot would let one user's typing burst (or a
|
||||||
|
/// different user's query) cancel another user's pending answer.
|
||||||
struct InlineDebounceState {
|
struct InlineDebounceState {
|
||||||
query: String,
|
query: String,
|
||||||
answered: bool,
|
answered: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
static INLINE_DEBOUNCE_STATE: LazyLock<parking_lot::Mutex<Option<InlineDebounceState>>> =
|
#[derive(Default)]
|
||||||
LazyLock::new(|| parking_lot::Mutex::new(None));
|
struct DebounceStates(HashMap<u64, InlineDebounceState>);
|
||||||
|
|
||||||
|
impl DebounceStates {
|
||||||
|
/// Records `query` as the user's newest query. Returns false when it is a
|
||||||
|
/// repeat whose answer already went out (Telegram's inline cache serves
|
||||||
|
/// it; re-fetching would only hit the source site again).
|
||||||
|
fn note(&mut self, user_id: u64, query: &str) -> bool {
|
||||||
|
if let Some(prev) = self.0.get(&user_id)
|
||||||
|
&& prev.query == query
|
||||||
|
&& prev.answered
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
self.0.insert(
|
||||||
|
user_id,
|
||||||
|
InlineDebounceState {
|
||||||
|
query: query.to_string(),
|
||||||
|
answered: false,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Claims the answer for the user's newest query; false when a newer query
|
||||||
|
/// superseded it or the answer was already claimed.
|
||||||
|
fn claim(&mut self, user_id: u64, query: &str) -> bool {
|
||||||
|
let Some(state) = self.0.get_mut(&user_id) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
if state.query != query || state.answered {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
state.answered = true;
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Releases a claimed-but-unsent answer so a repeat can retry the fetch.
|
||||||
|
fn release(&mut self, user_id: u64, query: &str) {
|
||||||
|
if let Some(state) = self.0.get_mut(&user_id)
|
||||||
|
&& state.query == query
|
||||||
|
{
|
||||||
|
state.answered = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static INLINE_DEBOUNCE_STATE: LazyLock<parking_lot::Mutex<DebounceStates>> =
|
||||||
|
LazyLock::new(|| parking_lot::Mutex::new(DebounceStates::default()));
|
||||||
|
|
||||||
pub async fn inline_query_handler(bot: Bot, query: InlineQuery) -> Result<(), RequestError> {
|
pub async fn inline_query_handler(bot: Bot, query: InlineQuery) -> Result<(), RequestError> {
|
||||||
if query.query.is_empty() {
|
if query.query.is_empty() {
|
||||||
@@ -41,48 +91,23 @@ pub async fn inline_query_handler(bot: Bot, query: InlineQuery) -> Result<(), Re
|
|||||||
// Debounce: record the query and answer only after it has been stable for
|
// Debounce: record the query and answer only after it has been stable for
|
||||||
// INLINE_DEBOUNCE (the timer below). An already-answered repeat of the
|
// INLINE_DEBOUNCE (the timer below). An already-answered repeat of the
|
||||||
// same query is left to Telegram's inline cache instead of re-fetching.
|
// same query is left to Telegram's inline cache instead of re-fetching.
|
||||||
{
|
let user_id = query.from.id.0;
|
||||||
let mut state = INLINE_DEBOUNCE_STATE.lock();
|
if !INLINE_DEBOUNCE_STATE.lock().note(user_id, &query.query) {
|
||||||
if let Some(prev) = state.as_ref()
|
return respond(());
|
||||||
&& prev.query == query.query
|
|
||||||
&& prev.answered
|
|
||||||
{
|
|
||||||
return respond(());
|
|
||||||
}
|
|
||||||
*state = Some(InlineDebounceState {
|
|
||||||
query: query.query.clone(),
|
|
||||||
answered: false,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
let query_text = query.query.clone();
|
let query_text = query.query.clone();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
tokio::time::sleep(INLINE_DEBOUNCE).await;
|
tokio::time::sleep(INLINE_DEBOUNCE).await;
|
||||||
// Only the last query of a typing burst survives: earlier timers see
|
// Only the user's last query of a typing burst survives: earlier
|
||||||
// the query changed and give up without answering.
|
// timers see the query changed and give up without answering.
|
||||||
{
|
if !INLINE_DEBOUNCE_STATE.lock().claim(user_id, &query_text) {
|
||||||
let mut state = INLINE_DEBOUNCE_STATE.lock();
|
return;
|
||||||
let Some(state) = state.as_mut() else {
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
if state.query != query_text || state.answered {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
// Claim the answer so a repeat of the same query cannot start a
|
|
||||||
// second fetch; reset below when no answer was produced.
|
|
||||||
state.answered = true;
|
|
||||||
}
|
}
|
||||||
match answer_inline_query(bot, query).await {
|
match answer_inline_query(bot, query).await {
|
||||||
Ok(true) => {}
|
Ok(true) => {}
|
||||||
// No results produced (or nothing to answer): let a repeat of the
|
// No results produced (or nothing to answer): let a repeat of the
|
||||||
// same query retry the fetch.
|
// same query retry the fetch.
|
||||||
Ok(false) | Err(_) => {
|
Ok(false) | Err(_) => INLINE_DEBOUNCE_STATE.lock().release(user_id, &query_text),
|
||||||
let mut state = INLINE_DEBOUNCE_STATE.lock();
|
|
||||||
if let Some(state) = state.as_mut()
|
|
||||||
&& state.query == query_text
|
|
||||||
{
|
|
||||||
state.answered = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
respond(())
|
respond(())
|
||||||
@@ -96,14 +121,40 @@ async fn answer_inline_query(bot: Bot, query: InlineQuery) -> Result<bool, Reque
|
|||||||
query.query,
|
query.query,
|
||||||
log_key(&query.query)
|
log_key(&query.query)
|
||||||
);
|
);
|
||||||
match x_media::site::fetch(&query.query).await {
|
// No retries: the debounce plus a 1s/2s backoff would outlast the inline
|
||||||
|
// query the answer belongs to.
|
||||||
|
match x_media::site::fetch_once(&query.query).await {
|
||||||
Ok(Some(fetched)) => {
|
Ok(Some(fetched)) => {
|
||||||
let mut results: Vec<InlineQueryResult> = Vec::new();
|
let mut results: Vec<InlineQueryResult> = Vec::new();
|
||||||
// Inline results have the same 1024-char caption limit as regular
|
// Inline results have the same 1024-char caption limit as regular
|
||||||
// messages; truncate once here for all items.
|
// messages; truncate once here for all items, then apply the same
|
||||||
|
// long-post quoting as the send paths. `answer_inline_query` has no
|
||||||
|
// `AppContext` (the debounce spawns it), so the parsed config comes
|
||||||
|
// from the process-wide static, and the text is the *escaped*
|
||||||
|
// title/content the built-in caption embeds (the raw
|
||||||
|
// `Fetched.title`/`content` differ whenever the post contains
|
||||||
|
// `<`/`&`).
|
||||||
let caption = x_media::site::truncate_caption(&fetched.caption);
|
let caption = x_media::site::truncate_caption(&fetched.caption);
|
||||||
|
let text = fetched
|
||||||
|
.render_fields()
|
||||||
|
.map(|(_, _, title, content, _)| x_media::site::compose_text(title, content))
|
||||||
|
.unwrap_or_default();
|
||||||
|
let caption = crate::send::quote_long_caption(
|
||||||
|
&caption,
|
||||||
|
&text,
|
||||||
|
super::CONFIG.caption_quote_text_chars,
|
||||||
|
);
|
||||||
for (i, media) in fetched.media.iter().enumerate() {
|
for (i, media) in fetched.media.iter().enumerate() {
|
||||||
let id = format!("{i}");
|
let id = format!("{i}");
|
||||||
|
// Telegram fetches an inline result's URL itself and cannot
|
||||||
|
// send site-specific headers, so hotlink-protected media
|
||||||
|
// (pixiv's pximg.net) would render as a broken file there.
|
||||||
|
// Locally produced media (ugoira MP4, bsky remux) is a local
|
||||||
|
// path and does not parse as a URL at all — same skip.
|
||||||
|
if x_media::site::needs_media_headers(media.url()) {
|
||||||
|
log::debug!("inline: skipping hotlink-protected media {id}");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
let Some(url) = url::Url::parse(media.url()).ok() else {
|
let Some(url) = url::Url::parse(media.url()).ok() else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
@@ -111,7 +162,7 @@ async fn answer_inline_query(bot: Bot, query: InlineQuery) -> Result<bool, Reque
|
|||||||
.thumbnail_url()
|
.thumbnail_url()
|
||||||
.and_then(|t| url::Url::parse(t).ok())
|
.and_then(|t| url::Url::parse(t).ok())
|
||||||
.unwrap_or_else(|| url.clone());
|
.unwrap_or_else(|| url.clone());
|
||||||
let caption = caption.clone();
|
let caption = caption.clone().into_owned();
|
||||||
let result = match media {
|
let result = match media {
|
||||||
Media::Illustration { .. } => {
|
Media::Illustration { .. } => {
|
||||||
// Inline photo results have their own (smaller) size
|
// Inline photo results have their own (smaller) size
|
||||||
@@ -159,3 +210,49 @@ async fn answer_inline_query(bot: Bot, query: InlineQuery) -> Result<bool, Reque
|
|||||||
}
|
}
|
||||||
Ok(false)
|
Ok(false)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::DebounceStates;
|
||||||
|
|
||||||
|
const URL_A: &str = "https://x.com/a/status/1";
|
||||||
|
const URL_B: &str = "https://x.com/b/status/2";
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn debounce_state_is_per_user() {
|
||||||
|
let mut states = DebounceStates::default();
|
||||||
|
// Two users query different links: both proceed, and neither timer
|
||||||
|
// cancels the other (a single shared slot dropped one of them).
|
||||||
|
assert!(states.note(1, URL_A));
|
||||||
|
assert!(states.note(2, URL_B));
|
||||||
|
assert!(states.claim(1, URL_A), "user 1's answer was cancelled");
|
||||||
|
assert!(states.claim(2, URL_B), "user 2's answer was cancelled");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn answered_query_is_suppressed_per_user_only() {
|
||||||
|
let mut states = DebounceStates::default();
|
||||||
|
assert!(states.note(1, URL_A));
|
||||||
|
assert!(states.claim(1, URL_A));
|
||||||
|
// A repeat of the answered query by the same user is left to
|
||||||
|
// Telegram's inline cache.
|
||||||
|
assert!(!states.note(1, URL_A));
|
||||||
|
// Another user pasting the same link still gets an answer.
|
||||||
|
assert!(states.note(2, URL_A));
|
||||||
|
assert!(states.claim(2, URL_A));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn newer_query_supersedes_and_failed_answer_is_released() {
|
||||||
|
let mut states = DebounceStates::default();
|
||||||
|
assert!(states.note(1, URL_A));
|
||||||
|
assert!(states.note(1, URL_B));
|
||||||
|
// The stale timer for the half-typed query gives up…
|
||||||
|
assert!(!states.claim(1, URL_A));
|
||||||
|
// …and the newest one answers.
|
||||||
|
assert!(states.claim(1, URL_B));
|
||||||
|
// No results → release so a repeat may retry the fetch.
|
||||||
|
states.release(1, URL_B);
|
||||||
|
assert!(states.claim(1, URL_B));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -18,25 +18,25 @@ pub use inline::inline_query_handler;
|
|||||||
pub use statics::{CHAT_STORE, CONFIG, LINK_CACHE, TASK_QUEUE};
|
pub use statics::{CHAT_STORE, CONFIG, LINK_CACHE, TASK_QUEUE};
|
||||||
pub use urls::{start_url_workers, stop_url_workers};
|
pub use urls::{start_url_workers, stop_url_workers};
|
||||||
|
|
||||||
|
use crate::ctx::AppContext;
|
||||||
use crate::media_sender::MediaSender;
|
use crate::media_sender::MediaSender;
|
||||||
use commands::{Command, execute_command};
|
use commands::{Command, execute_command};
|
||||||
use teloxide::RequestError;
|
use teloxide::RequestError;
|
||||||
use teloxide::prelude::*;
|
use teloxide::prelude::*;
|
||||||
use teloxide::types::{ChatId, ChatKind, Message, MessageId, ParseMode, ReplyParameters};
|
use teloxide::types::{
|
||||||
|
ChatId, ChatKind, Message, MessageId, ParseMode, PublicChatKind, ReplyParameters,
|
||||||
|
};
|
||||||
use teloxide::utils::command::BotCommands;
|
use teloxide::utils::command::BotCommands;
|
||||||
use urls::{URL_JOBS, extract_urls};
|
use urls::{URL_JOBS, extract_urls};
|
||||||
|
|
||||||
/// Reply to a message by id, keeping the reply decoration even if the
|
/// Reply to a message by id, keeping the reply decoration even if the
|
||||||
/// original was already deleted.
|
/// original was already deleted. Returns the reply's message id.
|
||||||
pub(crate) async fn reply<T>(
|
pub(crate) async fn reply(
|
||||||
sender: &dyn MediaSender,
|
sender: &dyn MediaSender,
|
||||||
chat_id: i64,
|
chat_id: i64,
|
||||||
reply_to: MessageId,
|
reply_to: MessageId,
|
||||||
text: T,
|
text: impl Into<String>,
|
||||||
) -> Result<Message, RequestError>
|
) -> Result<i64, RequestError> {
|
||||||
where
|
|
||||||
T: Into<String>,
|
|
||||||
{
|
|
||||||
sender
|
sender
|
||||||
.send_message(ChatId(chat_id), text.into(), Some(reply_to), None)
|
.send_message(ChatId(chat_id), text.into(), Some(reply_to), None)
|
||||||
.await
|
.await
|
||||||
@@ -50,35 +50,36 @@ pub(crate) async fn reply_html(
|
|||||||
chat_id: i64,
|
chat_id: i64,
|
||||||
reply_to: MessageId,
|
reply_to: MessageId,
|
||||||
text: String,
|
text: String,
|
||||||
) -> Result<Message, RequestError> {
|
) -> Result<i64, RequestError> {
|
||||||
// `<Bot as Requester>::` disambiguates from the MediaSender trait's
|
// `<Bot as Requester>::` disambiguates from the MediaSender trait's
|
||||||
// same-named method (see media_sender.rs).
|
// same-named method (see media_sender.rs).
|
||||||
<Bot as Requester>::send_message(bot, ChatId(chat_id), text)
|
<Bot as Requester>::send_message(bot, ChatId(chat_id), text)
|
||||||
.parse_mode(ParseMode::Html)
|
.parse_mode(ParseMode::Html)
|
||||||
.reply_parameters(ReplyParameters::new(reply_to).allow_sending_without_reply())
|
.reply_parameters(ReplyParameters::new(reply_to).allow_sending_without_reply())
|
||||||
.await
|
.await
|
||||||
|
.map(|message| message.id.0 as i64)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Log prefix tying the whole lifecycle of one link (fetch → send → cache →
|
/// Log prefix tying the whole lifecycle of one link (fetch → send → cache →
|
||||||
/// forward) together: the normalized cache key (`twitter:123…`, `pixiv:123`,
|
/// forward) together: the normalized cache key (`twitter:123…`, `pixiv:123`,
|
||||||
/// `bsky:handle/rkey`) instead of the raw URL, so logs stay short and do not
|
/// `bsky:handle/rkey`, `bilibili:123…`) instead of the raw URL, so logs stay
|
||||||
/// echo full user-submitted URLs at info level.
|
/// short and do not echo full user-submitted URLs at info level.
|
||||||
pub fn log_key(url: &str) -> String {
|
pub fn log_key(url: &str) -> String {
|
||||||
x_media::site::cache_key(url).unwrap_or_else(|| "<unsupported>".to_string())
|
x_media::site::cache_key(url).unwrap_or_else(|| "<unsupported>".to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Edit-before-forward: a reply to the prompt swaps the caption of the first
|
/// Edit-before-forward: a reply to the prompt swaps the caption of the first
|
||||||
/// forwarded message. Returns true when the message was consumed as an edit.
|
/// forwarded message. Returns true when the message was consumed as an edit.
|
||||||
async fn edit_message_handler(bot: &Bot, message: &Message) -> bool {
|
/// Body of [`message_handler`]'s edit branch, without teloxide update types so
|
||||||
let Some(reply) = message.reply_to_message() else {
|
/// it can be driven by tests.
|
||||||
return false;
|
async fn edit_message_handler(
|
||||||
};
|
ctx: &AppContext<'_>,
|
||||||
let chat_id = message.chat.id.0;
|
chat_id: i64,
|
||||||
let Some(text) = message.text() else {
|
reply_to_message_id: i64,
|
||||||
return false;
|
text: &str,
|
||||||
};
|
) -> bool {
|
||||||
let chat_data = CHAT_STORE.get(chat_id).await;
|
let chat_data = ctx.chat_store.get(chat_id).await;
|
||||||
let Some(edit) = chat_data.edit_message.get(&(reply.id.0 as i64)) else {
|
let Some(edit) = chat_data.edit_message.get(&reply_to_message_id) else {
|
||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
let Some(first_forward_id) = edit.forward_message_ids.first() else {
|
let Some(first_forward_id) = edit.forward_message_ids.first() else {
|
||||||
@@ -98,15 +99,17 @@ async fn edit_message_handler(bot: &Bot, message: &Message) -> bool {
|
|||||||
.map(|template| template.replace("[]", &link))
|
.map(|template| template.replace("[]", &link))
|
||||||
.unwrap_or(link)
|
.unwrap_or(link)
|
||||||
};
|
};
|
||||||
let result = bot
|
match ctx
|
||||||
.edit_message_caption(ChatId(chat_id), MessageId(*first_forward_id as i32))
|
.sender
|
||||||
.caption(new_text)
|
.edit_message_caption(
|
||||||
.parse_mode(ParseMode::Html)
|
ChatId(chat_id),
|
||||||
.await;
|
MessageId(*first_forward_id as i32),
|
||||||
match result {
|
new_text,
|
||||||
Ok(_) => log::info!(
|
)
|
||||||
"edit-before-forward: caption swapped on message {first_forward_id} for prompt {}",
|
.await
|
||||||
reply.id.0
|
{
|
||||||
|
Ok(()) => log::info!(
|
||||||
|
"edit-before-forward: caption swapped on message {first_forward_id} for prompt {reply_to_message_id}"
|
||||||
),
|
),
|
||||||
Err(e) => log::error!("edit_message_caption failed: {e}"),
|
Err(e) => log::error!("edit_message_caption failed: {e}"),
|
||||||
}
|
}
|
||||||
@@ -133,7 +136,17 @@ pub async fn message_handler(bot: Bot, message: Message) -> Result<(), RequestEr
|
|||||||
message.chat.id
|
message.chat.id
|
||||||
);
|
);
|
||||||
// URL/edit flows only run in private chats; commands run in any chat.
|
// URL/edit flows only run in private chats; commands run in any chat.
|
||||||
if is_private && edit_message_handler(&bot, &message).await {
|
if is_private
|
||||||
|
&& let Some(reply) = message.reply_to_message()
|
||||||
|
&& let Some(text) = message.text()
|
||||||
|
&& edit_message_handler(
|
||||||
|
&AppContext::from_statics(&bot),
|
||||||
|
message.chat.id.0,
|
||||||
|
reply.id.0 as i64,
|
||||||
|
text,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
return respond(());
|
return respond(());
|
||||||
}
|
}
|
||||||
if let Some(text) = message.text()
|
if let Some(text) = message.text()
|
||||||
@@ -157,8 +170,168 @@ pub async fn message_handler(bot: Bot, message: Message) -> Result<(), RequestEr
|
|||||||
log::warn!("url workers not started; dropping link");
|
log::warn!("url workers not started; dropping link");
|
||||||
break;
|
break;
|
||||||
};
|
};
|
||||||
let _ = tx.send((message.clone(), url)).await;
|
// A closed channel means the workers are stopping (shutdown):
|
||||||
|
// report the dropped link instead of losing it silently.
|
||||||
|
if tx.send((message.clone(), url)).await.is_err() {
|
||||||
|
log::warn!("url workers stopped; dropping link");
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
} else if is_group(&message.chat.kind)
|
||||||
|
&& extract_urls(&message)
|
||||||
|
.iter()
|
||||||
|
.any(|url| x_media::site::cache_key(url).is_some())
|
||||||
|
{
|
||||||
|
// A supported link in a group used to be dropped in silence, which
|
||||||
|
// reads as a broken bot (the command menu is registered globally, so
|
||||||
|
// the expectation is there). Unsupported links stay ignored; the hint
|
||||||
|
// names the two paths that do work. Channels are excluded — the reply
|
||||||
|
// would be posted into the channel itself.
|
||||||
|
let _ = reply(&bot, message.chat.id.0, message.id, GROUP_LINK_HINT).await;
|
||||||
}
|
}
|
||||||
respond(())
|
respond(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Answer for a link posted where the pipeline does not run (a group): links
|
||||||
|
/// are private-chat only, inline mode is the group path.
|
||||||
|
const GROUP_LINK_HINT: &str =
|
||||||
|
"Links are handled in private chat only — send me this link there, or use inline mode here.";
|
||||||
|
|
||||||
|
/// Groups and supergroups, as opposed to private chats and channels.
|
||||||
|
fn is_group(kind: &ChatKind) -> bool {
|
||||||
|
matches!(
|
||||||
|
kind,
|
||||||
|
ChatKind::Public(chat)
|
||||||
|
if matches!(
|
||||||
|
chat.kind,
|
||||||
|
PublicChatKind::Group | PublicChatKind::Supergroup(_)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::ctx::test_support::TestStores;
|
||||||
|
use crate::media_sender::test_support::{MockSender, Outcome};
|
||||||
|
use crate::state::EditMessage;
|
||||||
|
use teloxide::ApiError;
|
||||||
|
|
||||||
|
const PROMPT_ID: i64 = 7;
|
||||||
|
const FORWARDED_ID: i64 = 9;
|
||||||
|
|
||||||
|
fn api_error() -> RequestError {
|
||||||
|
RequestError::Api(ApiError::Unknown("Bad Request: message not found".into()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Seeds a prompt record; `template` names the chat template used for it
|
||||||
|
/// (empty = none, the caption gets the bare link).
|
||||||
|
async fn seed_prompt(ctx: &AppContext<'_>, template: &str) {
|
||||||
|
ctx.chat_store
|
||||||
|
.update(1, |data| {
|
||||||
|
data.template
|
||||||
|
.insert("tpl".to_string(), "<b>[]</b>".to_string());
|
||||||
|
data.edit_message.insert(
|
||||||
|
PROMPT_ID,
|
||||||
|
EditMessage {
|
||||||
|
url: "https://x.com/u/status/1".into(),
|
||||||
|
chat_id: 1,
|
||||||
|
forward_message_ids: vec![FORWARDED_ID],
|
||||||
|
template: template.to_string(),
|
||||||
|
created_at: crate::db::unix_now(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn reply_to_a_prompt_swaps_the_caption_through_its_template() {
|
||||||
|
let sender = MockSender::scripted(vec![Outcome::EditOk], api_error);
|
||||||
|
let stores = TestStores::new();
|
||||||
|
let ctx = stores.ctx(&sender);
|
||||||
|
seed_prompt(&ctx, "tpl").await;
|
||||||
|
|
||||||
|
let consumed = edit_message_handler(&ctx, 1, PROMPT_ID, "new caption").await;
|
||||||
|
|
||||||
|
assert!(consumed, "a reply to the prompt must be consumed");
|
||||||
|
assert_eq!(
|
||||||
|
sender.captions(),
|
||||||
|
vec!["<b><a href=\"https://x.com/u/status/1\">new caption</a></b>"]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn reply_text_and_url_are_escaped_into_the_caption() {
|
||||||
|
let sender = MockSender::scripted(vec![Outcome::EditOk], api_error);
|
||||||
|
let stores = TestStores::new();
|
||||||
|
let ctx = stores.ctx(&sender);
|
||||||
|
seed_prompt(&ctx, "").await;
|
||||||
|
|
||||||
|
edit_message_handler(&ctx, 1, PROMPT_ID, "<script>alert(1)</script>").await;
|
||||||
|
|
||||||
|
// No raw markup from user text may reach the HTML caption.
|
||||||
|
assert_eq!(
|
||||||
|
sender.captions(),
|
||||||
|
vec!["<a href=\"https://x.com/u/status/1\"><script>alert(1)</script></a>"]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_failed_caption_swap_still_consumes_the_reply() {
|
||||||
|
let sender = MockSender::scripted(vec![Outcome::EditErr], api_error);
|
||||||
|
let stores = TestStores::new();
|
||||||
|
let ctx = stores.ctx(&sender);
|
||||||
|
seed_prompt(&ctx, "tpl").await;
|
||||||
|
|
||||||
|
// The edit failed (message deleted etc.); the reply must still be
|
||||||
|
// swallowed instead of being treated as a link to fetch.
|
||||||
|
assert!(edit_message_handler(&ctx, 1, PROMPT_ID, "new caption").await);
|
||||||
|
assert_eq!(sender.calls(), vec!["edit_message_caption"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn reply_to_an_unrelated_message_is_not_consumed() {
|
||||||
|
let sender = MockSender::scripted(vec![], api_error);
|
||||||
|
let stores = TestStores::new();
|
||||||
|
let ctx = stores.ctx(&sender);
|
||||||
|
|
||||||
|
// No prompt record for that message id → the reply runs the normal
|
||||||
|
// (URL/command) path instead.
|
||||||
|
assert!(!edit_message_handler(&ctx, 1, PROMPT_ID, "hello").await);
|
||||||
|
assert!(sender.calls().is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_link_hint_is_for_groups_only() {
|
||||||
|
use teloxide::types::{ChatPrivate, ChatPublic, PublicChatChannel, PublicChatSupergroup};
|
||||||
|
|
||||||
|
let group = ChatKind::Public(ChatPublic {
|
||||||
|
title: None,
|
||||||
|
kind: PublicChatKind::Group,
|
||||||
|
});
|
||||||
|
let supergroup = ChatKind::Public(ChatPublic {
|
||||||
|
title: None,
|
||||||
|
kind: PublicChatKind::Supergroup(PublicChatSupergroup {
|
||||||
|
username: None,
|
||||||
|
is_forum: false,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
// A channel must stay silent: the hint reply would be posted into the
|
||||||
|
// channel itself.
|
||||||
|
let channel = ChatKind::Public(ChatPublic {
|
||||||
|
title: None,
|
||||||
|
kind: PublicChatKind::Channel(PublicChatChannel { username: None }),
|
||||||
|
});
|
||||||
|
let private = ChatKind::Private(ChatPrivate {
|
||||||
|
username: None,
|
||||||
|
first_name: None,
|
||||||
|
last_name: None,
|
||||||
|
});
|
||||||
|
|
||||||
|
assert!(is_group(&group));
|
||||||
|
assert!(is_group(&supergroup));
|
||||||
|
assert!(!is_group(&channel));
|
||||||
|
assert!(!is_group(&private));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,15 +1,14 @@
|
|||||||
//! URL extraction and the per-URL media pipeline: bounded job channel +
|
//! URL extraction and the per-URL media pipeline: bounded job channel +
|
||||||
//! worker pool, link-cache fast path, fetch, task build and send dispatch.
|
//! worker pool, link-cache fast path, fetch, task build and send dispatch.
|
||||||
|
|
||||||
use super::{CHAT_STORE, CONFIG, LINK_CACHE, TASK_QUEUE, log_key, reply};
|
use super::{log_key, reply};
|
||||||
use crate::config::Config;
|
use crate::ctx::{AppContext, CONTEXT};
|
||||||
use crate::db::now_f64;
|
use crate::link_cache::{CachedMediaKind, CachedPost};
|
||||||
use crate::link_cache::{CachedMediaKind, CachedPost, LinkCache};
|
|
||||||
use crate::media_sender::MediaSender;
|
use crate::media_sender::MediaSender;
|
||||||
use crate::queue::PersistentTaskQueue;
|
|
||||||
use crate::send::{self, MediaItemPayload, Task};
|
use crate::send::{self, MediaItemPayload, Task};
|
||||||
use crate::state::{ChatData, ChatStore};
|
use crate::state::ChatData;
|
||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
|
use std::future::Future;
|
||||||
use std::sync::LazyLock;
|
use std::sync::LazyLock;
|
||||||
use teloxide::types::{ChatAction, ChatId, Message, MessageEntityKind, MessageId};
|
use teloxide::types::{ChatAction, ChatId, Message, MessageEntityKind, MessageId};
|
||||||
use x_media::media::Media;
|
use x_media::media::Media;
|
||||||
@@ -35,27 +34,6 @@ static URL_WORKER_HANDLES: LazyLock<parking_lot::Mutex<Option<Vec<tokio::task::J
|
|||||||
/// while bounding how many jobs can be queued at all.
|
/// while bounding how many jobs can be queued at all.
|
||||||
const URL_WORKERS: usize = 8;
|
const URL_WORKERS: usize = 8;
|
||||||
|
|
||||||
/// Dependencies of the per-URL pipeline, injected so tests can substitute a
|
|
||||||
/// mock sender and tempdir-backed stores.
|
|
||||||
pub(crate) struct AppContext<'a> {
|
|
||||||
pub sender: &'a dyn MediaSender,
|
|
||||||
pub chat_store: &'a ChatStore,
|
|
||||||
pub task_queue: &'a PersistentTaskQueue,
|
|
||||||
pub link_cache: &'a LinkCache,
|
|
||||||
pub config: &'a Config,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Assembles the production context from the process-wide statics.
|
|
||||||
fn app_context() -> AppContext<'static> {
|
|
||||||
AppContext {
|
|
||||||
sender: &*crate::send::BOT,
|
|
||||||
chat_store: &CHAT_STORE,
|
|
||||||
task_queue: &TASK_QUEUE,
|
|
||||||
link_cache: &LINK_CACHE,
|
|
||||||
config: &CONFIG,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Starts the URL job workers (called once from main after the queue starts).
|
/// Starts the URL job workers (called once from main after the queue starts).
|
||||||
/// teloxide dispatches updates to a per-chat worker that handles them
|
/// teloxide dispatches updates to a per-chat worker that handles them
|
||||||
/// sequentially, so a batch-forward of many messages would otherwise be
|
/// sequentially, so a batch-forward of many messages would otherwise be
|
||||||
@@ -70,12 +48,18 @@ pub async fn start_url_workers() {
|
|||||||
for _ in 0..URL_WORKERS {
|
for _ in 0..URL_WORKERS {
|
||||||
let rx = std::sync::Arc::clone(&rx);
|
let rx = std::sync::Arc::clone(&rx);
|
||||||
handles.push(tokio::spawn(async move {
|
handles.push(tokio::spawn(async move {
|
||||||
let ctx = app_context();
|
|
||||||
while !URL_STOP.load(std::sync::atomic::Ordering::Relaxed) {
|
while !URL_STOP.load(std::sync::atomic::Ordering::Relaxed) {
|
||||||
let job = rx.lock().await.recv().await;
|
let job = rx.lock().await.recv().await;
|
||||||
match job {
|
match job {
|
||||||
Some((message, url)) => {
|
Some((message, url)) => {
|
||||||
url_media(&ctx, message.chat.id.0, message.id.0 as i64, &url).await
|
url_media(
|
||||||
|
&CONTEXT,
|
||||||
|
message.chat.id.0,
|
||||||
|
message.id.0 as i64,
|
||||||
|
&url,
|
||||||
|
PostSend::FromChat,
|
||||||
|
)
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
None => break,
|
None => break,
|
||||||
}
|
}
|
||||||
@@ -174,14 +158,6 @@ fn media_to_payload(media: &Media, sensitive: bool) -> MediaItemPayload {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn enqueue_retry(queue: &PersistentTaskQueue, task: Task, delay_seconds: f64) {
|
|
||||||
let payload = serde_json::to_value(task).expect("task serializes");
|
|
||||||
let run_after = now_f64() + delay_seconds;
|
|
||||||
if let Err(e) = queue.enqueue(payload, run_after).await {
|
|
||||||
log::error!("failed to enqueue retry: {e}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Sends a task and handles the outcome: post-send actions on success, retry
|
/// Sends a task and handles the outcome: post-send actions on success, retry
|
||||||
/// enqueue on retryable failure, reply + link-cache invalidation on
|
/// enqueue on retryable failure, reply + link-cache invalidation on
|
||||||
/// permanent failure (a stale cached file id must not repeat forever).
|
/// permanent failure (a stale cached file id must not repeat forever).
|
||||||
@@ -193,8 +169,8 @@ async fn dispatch_send(
|
|||||||
url: &str,
|
url: &str,
|
||||||
) {
|
) {
|
||||||
let result = match task {
|
let result = match task {
|
||||||
Task::SendAnimation { .. } => send::send_animation(ctx.sender, task).await,
|
Task::SendAnimation { .. } => send::send_animation(ctx, task).await,
|
||||||
Task::SendMediaSequence { .. } => send::send_media_sequence(ctx.sender, task).await,
|
Task::SendMediaSequence { .. } => send::send_media_sequence(ctx, task).await,
|
||||||
Task::ForwardMessages { .. } => unreachable!(),
|
Task::ForwardMessages { .. } => unreachable!(),
|
||||||
};
|
};
|
||||||
match result {
|
match result {
|
||||||
@@ -204,9 +180,8 @@ async fn dispatch_send(
|
|||||||
message_ids.len(),
|
message_ids.len(),
|
||||||
log_key(url)
|
log_key(url)
|
||||||
);
|
);
|
||||||
send::post_send_actions(ctx.sender, task, message_ids).await;
|
send::post_send_actions(ctx, task, message_ids).await;
|
||||||
// The task settled: drop any keep-alive temp media.
|
send::settle_task(ctx, task, send::Settled::Sent).await;
|
||||||
send::release_keep_alive(task);
|
|
||||||
}
|
}
|
||||||
Err(send::SendError::Retryable {
|
Err(send::SendError::Retryable {
|
||||||
delay_seconds,
|
delay_seconds,
|
||||||
@@ -216,12 +191,17 @@ async fn dispatch_send(
|
|||||||
"send for [key={}] failed, queued for retry in {delay_seconds:.1}s",
|
"send for [key={}] failed, queued for retry in {delay_seconds:.1}s",
|
||||||
log_key(url)
|
log_key(url)
|
||||||
);
|
);
|
||||||
enqueue_retry(ctx.task_queue, *task, delay_seconds).await;
|
send::enqueue_retry(ctx.task_queue, *task, delay_seconds).await;
|
||||||
|
// Name the post and the wait: "queued for retry" alone left the
|
||||||
|
// user guessing which link it was and how long the wait is.
|
||||||
let _ = reply(
|
let _ = reply(
|
||||||
ctx.sender,
|
ctx.sender,
|
||||||
chat_id,
|
chat_id,
|
||||||
reply_to,
|
reply_to,
|
||||||
"Send failed. Task queued for retry.",
|
format!(
|
||||||
|
"Send failed for {} — retrying in {delay_seconds:.0}s.",
|
||||||
|
log_key(url)
|
||||||
|
),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
@@ -229,8 +209,7 @@ async fn dispatch_send(
|
|||||||
message: err_message,
|
message: err_message,
|
||||||
task,
|
task,
|
||||||
}) => {
|
}) => {
|
||||||
send::invalidate_cache_with(ctx.link_cache, &task).await;
|
send::settle_task(ctx, &task, send::Settled::Failed).await;
|
||||||
send::release_keep_alive(&task);
|
|
||||||
log::error!("send for {url} failed permanently: {err_message}");
|
log::error!("send for {url} failed permanently: {err_message}");
|
||||||
let _ = reply(
|
let _ = reply(
|
||||||
ctx.sender,
|
ctx.sender,
|
||||||
@@ -243,8 +222,20 @@ async fn dispatch_send(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Whether a send also runs the chat's post-send actions. `/test` sends with
|
||||||
|
/// them suppressed so a test can never forward to the channel or open the
|
||||||
|
/// edit-before-forward prompt; a normal link uses whatever the chat is
|
||||||
|
/// configured with.
|
||||||
|
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
|
||||||
|
pub(crate) enum PostSend {
|
||||||
|
/// Apply the chat's `forward_channel_id` / `edit_before_forward`.
|
||||||
|
FromChat,
|
||||||
|
/// Send only: no channel forward, no edit prompt.
|
||||||
|
Suppressed,
|
||||||
|
}
|
||||||
|
|
||||||
/// Builds the send task from ready-made items, sharing the payload shape
|
/// Builds the send task from ready-made items, sharing the payload shape
|
||||||
/// between the fresh-fetch and link-cache paths.
|
/// between the fresh-fetch, link-cache and `/test` paths.
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
fn build_send_task(
|
fn build_send_task(
|
||||||
chat_data: &ChatData,
|
chat_data: &ChatData,
|
||||||
@@ -254,7 +245,14 @@ fn build_send_task(
|
|||||||
caption: String,
|
caption: String,
|
||||||
items: Vec<MediaItemPayload>,
|
items: Vec<MediaItemPayload>,
|
||||||
cache_data: Option<CachedPost>,
|
cache_data: Option<CachedPost>,
|
||||||
|
post_send: PostSend,
|
||||||
) -> Task {
|
) -> Task {
|
||||||
|
// Notification ids stay set in both modes: a queued retry that
|
||||||
|
// dead-letters should still tell the chat.
|
||||||
|
let (edit_before_forward, forward_channel_id) = match post_send {
|
||||||
|
PostSend::FromChat => (chat_data.edit_before_forward, chat_data.forward_channel_id),
|
||||||
|
PostSend::Suppressed => (false, None),
|
||||||
|
};
|
||||||
if items.len() == 1 && matches!(items[0], MediaItemPayload::Animation { .. }) {
|
if items.len() == 1 && matches!(items[0], MediaItemPayload::Animation { .. }) {
|
||||||
Task::SendAnimation {
|
Task::SendAnimation {
|
||||||
chat_id,
|
chat_id,
|
||||||
@@ -262,8 +260,8 @@ fn build_send_task(
|
|||||||
caption,
|
caption,
|
||||||
animation: items.into_iter().next().unwrap(),
|
animation: items.into_iter().next().unwrap(),
|
||||||
source_url,
|
source_url,
|
||||||
edit_before_forward: chat_data.edit_before_forward,
|
edit_before_forward,
|
||||||
forward_channel_id: chat_data.forward_channel_id,
|
forward_channel_id,
|
||||||
notify_chat_id: Some(chat_id),
|
notify_chat_id: Some(chat_id),
|
||||||
notify_message_id: Some(reply_to_message_id),
|
notify_message_id: Some(reply_to_message_id),
|
||||||
cache_data,
|
cache_data,
|
||||||
@@ -279,8 +277,8 @@ fn build_send_task(
|
|||||||
batch_index: 0,
|
batch_index: 0,
|
||||||
sent_message_ids: vec![],
|
sent_message_ids: vec![],
|
||||||
source_url,
|
source_url,
|
||||||
edit_before_forward: chat_data.edit_before_forward,
|
edit_before_forward,
|
||||||
forward_channel_id: chat_data.forward_channel_id,
|
forward_channel_id,
|
||||||
notify_chat_id: Some(chat_id),
|
notify_chat_id: Some(chat_id),
|
||||||
notify_message_id: Some(reply_to_message_id),
|
notify_message_id: Some(reply_to_message_id),
|
||||||
cache_data,
|
cache_data,
|
||||||
@@ -288,15 +286,154 @@ fn build_send_task(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn url_media(ctx: &AppContext<'_>, chat_id: i64, reply_to_message_id: i64, url: &str) {
|
/// The per-URL pipeline: link cache → fetch → build → send → post-send.
|
||||||
let reply_to = MessageId(reply_to_message_id as i32);
|
///
|
||||||
if let Err(e) = ctx
|
/// `post_send` selects whether the chat's forward/edit settings apply: the URL
|
||||||
.sender
|
/// workers pass [`PostSend::FromChat`], the `/test` command
|
||||||
.send_chat_action(ChatId(chat_id), ChatAction::Typing)
|
/// [`PostSend::Suppressed`]. Everything else (cache write, retry enqueue,
|
||||||
.await
|
/// dead-letter notification) is identical.
|
||||||
{
|
///
|
||||||
|
/// Wraps [`url_media_inner`] with the chat-action keep-alive: Telegram expires
|
||||||
|
/// an action indicator after ~5s, while a fetch (ugoira encode, HLS remux) plus
|
||||||
|
/// a download-and-reupload fallback routinely takes longer — without the
|
||||||
|
/// refresh the chat shows nothing and the bot reads as stalled.
|
||||||
|
pub(crate) async fn url_media(
|
||||||
|
ctx: &AppContext<'_>,
|
||||||
|
chat_id: i64,
|
||||||
|
reply_to_message_id: i64,
|
||||||
|
url: &str,
|
||||||
|
post_send: PostSend,
|
||||||
|
) {
|
||||||
|
// Shared with the pipeline: once the media types are known the indicator
|
||||||
|
// switches from "typing" to "sending photo/video".
|
||||||
|
let hint = parking_lot::Mutex::new(ActionHint::Typing);
|
||||||
|
run_with_chat_action(
|
||||||
|
ctx.sender,
|
||||||
|
chat_id,
|
||||||
|
&hint,
|
||||||
|
url_media_inner(ctx, chat_id, reply_to_message_id, url, post_send, &hint),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Runs `pipeline` while keeping the chat's action indicator alive: Telegram
|
||||||
|
/// expires an action after ~5s, while a fetch (ugoira encode, HLS remux) plus a
|
||||||
|
/// download-and-reupload fallback routinely takes longer. The pipeline updates
|
||||||
|
/// `hint` when it knows what it is sending.
|
||||||
|
async fn run_with_chat_action<F: Future<Output = ()>>(
|
||||||
|
sender: &dyn MediaSender,
|
||||||
|
chat_id: i64,
|
||||||
|
hint: &parking_lot::Mutex<ActionHint>,
|
||||||
|
pipeline: F,
|
||||||
|
) {
|
||||||
|
// The guard is released before the await: a parking_lot guard held across
|
||||||
|
// it makes the future !Send, and the URL workers spawn these.
|
||||||
|
let action = hint.lock().action();
|
||||||
|
if let Err(e) = sender.send_chat_action(ChatId(chat_id), action).await {
|
||||||
log::error!("send_chat_action failed: {e}");
|
log::error!("send_chat_action failed: {e}");
|
||||||
}
|
}
|
||||||
|
tokio::pin!(pipeline);
|
||||||
|
loop {
|
||||||
|
tokio::select! {
|
||||||
|
// `biased` polls the pipeline first, so a finished pipeline returns
|
||||||
|
// without ever arming the refresh timer (no stray actions).
|
||||||
|
biased;
|
||||||
|
() = &mut pipeline => return,
|
||||||
|
() = tokio::time::sleep(ACTION_REFRESH) => {
|
||||||
|
let action = hint.lock().action();
|
||||||
|
if let Err(e) = sender.send_chat_action(ChatId(chat_id), action).await {
|
||||||
|
log::error!("send_chat_action failed: {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// How often the chat-action indicator is refreshed while a pipeline runs.
|
||||||
|
/// Telegram's indicator lasts ~5s; refreshing slightly inside that keeps it
|
||||||
|
/// on-screen continuously.
|
||||||
|
const ACTION_REFRESH: std::time::Duration = std::time::Duration::from_secs(4);
|
||||||
|
|
||||||
|
/// What the chat action should say. Unknown before the fetch, so the pipeline
|
||||||
|
/// starts with `Typing` and switches as soon as the media types are known.
|
||||||
|
#[derive(Clone, Copy)]
|
||||||
|
enum ActionHint {
|
||||||
|
Typing,
|
||||||
|
Photo,
|
||||||
|
Video,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ActionHint {
|
||||||
|
/// Photos make Telegram label the send "sending photo"; video/animation
|
||||||
|
/// only payloads get "sending video". A mixed post takes the photo label
|
||||||
|
/// (the group's first item is always a photo, see `photos_first`).
|
||||||
|
fn for_items(items: &[MediaItemPayload]) -> Self {
|
||||||
|
if items
|
||||||
|
.iter()
|
||||||
|
.any(|item| matches!(item, MediaItemPayload::Photo { .. }))
|
||||||
|
{
|
||||||
|
Self::Photo
|
||||||
|
} else {
|
||||||
|
Self::Video
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn action(self) -> ChatAction {
|
||||||
|
match self {
|
||||||
|
Self::Typing => ChatAction::Typing,
|
||||||
|
Self::Photo => ChatAction::UploadPhoto,
|
||||||
|
Self::Video => ChatAction::UploadVideo,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// User-facing text for a failed fetch. The [`FetchError`] class is what tells
|
||||||
|
/// the user whether the post is gone, withheld or the source is refusing
|
||||||
|
/// requests; a single generic sentence threw that away.
|
||||||
|
fn fetch_error_message(err: &x_media::site::FetchError) -> String {
|
||||||
|
use x_media::site::FetchError;
|
||||||
|
match err {
|
||||||
|
FetchError::NotFound => "Post not found (deleted, private or unavailable).".to_string(),
|
||||||
|
FetchError::Sensitive => concat!(
|
||||||
|
"This post's media is withheld (age-restricted). ",
|
||||||
|
"The bot owner must set TWITTER_AUTH_TOKEN to fetch it."
|
||||||
|
)
|
||||||
|
.to_string(),
|
||||||
|
FetchError::Blocked => {
|
||||||
|
"The source site refused the request (risk control). Try again later.".to_string()
|
||||||
|
}
|
||||||
|
FetchError::Disabled { site } => {
|
||||||
|
format!("{} support is disabled on this bot.", site_title(site))
|
||||||
|
}
|
||||||
|
FetchError::Transient(_) | FetchError::Http(_) => {
|
||||||
|
"The source site is unavailable right now (tried 3 times). Try again later.".to_string()
|
||||||
|
}
|
||||||
|
// Parse/shape surprises, pixiv auth details, oversized media: nothing
|
||||||
|
// actionable for the user beyond "this did not work".
|
||||||
|
_ => "Failed to fetch media from this link.".to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Site ids are lowercase ASCII (`pixiv`); user-facing text capitalizes the
|
||||||
|
/// first letter.
|
||||||
|
fn site_title(site: &str) -> String {
|
||||||
|
let mut chars = site.chars();
|
||||||
|
match chars.next() {
|
||||||
|
Some(first) => first.to_uppercase().collect::<String>() + chars.as_str(),
|
||||||
|
None => String::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
async fn url_media_inner(
|
||||||
|
ctx: &AppContext<'_>,
|
||||||
|
chat_id: i64,
|
||||||
|
reply_to_message_id: i64,
|
||||||
|
url: &str,
|
||||||
|
post_send: PostSend,
|
||||||
|
hint: &parking_lot::Mutex<ActionHint>,
|
||||||
|
) {
|
||||||
|
let reply_to = MessageId(reply_to_message_id as i32);
|
||||||
|
|
||||||
// Link cache: a post sent before is re-sent from Telegram file ids —
|
// Link cache: a post sent before is re-sent from Telegram file ids —
|
||||||
// no source-site request, no download, no upload. Keyed by the
|
// no source-site request, no download, no upload. Keyed by the
|
||||||
@@ -324,6 +461,7 @@ async fn url_media(ctx: &AppContext<'_>, chat_id: i64, reply_to_message_id: i64,
|
|||||||
&cached.author,
|
&cached.author,
|
||||||
&cached.author_url,
|
&cached.author_url,
|
||||||
&cached.title,
|
&cached.title,
|
||||||
|
&cached.content,
|
||||||
&cached.tags,
|
&cached.tags,
|
||||||
)
|
)
|
||||||
};
|
};
|
||||||
@@ -351,6 +489,9 @@ async fn url_media(ctx: &AppContext<'_>, chat_id: i64, reply_to_message_id: i64,
|
|||||||
},
|
},
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
// The indicator switches to "sending photo/video" once the kinds are
|
||||||
|
// known; `items` is moved into the task below.
|
||||||
|
*hint.lock() = ActionHint::for_items(&items);
|
||||||
let task = build_send_task(
|
let task = build_send_task(
|
||||||
&chat_data,
|
&chat_data,
|
||||||
chat_id,
|
chat_id,
|
||||||
@@ -359,6 +500,7 @@ async fn url_media(ctx: &AppContext<'_>, chat_id: i64, reply_to_message_id: i64,
|
|||||||
caption,
|
caption,
|
||||||
items,
|
items,
|
||||||
Some(cached),
|
Some(cached),
|
||||||
|
post_send,
|
||||||
);
|
);
|
||||||
dispatch_send(ctx, chat_id, reply_to, &task, url).await;
|
dispatch_send(ctx, chat_id, reply_to, &task, url).await;
|
||||||
return;
|
return;
|
||||||
@@ -373,13 +515,7 @@ async fn url_media(ctx: &AppContext<'_>, chat_id: i64, reply_to_message_id: i64,
|
|||||||
// Retries exhausted: notify the user (Rust-only requirement 3).
|
// Retries exhausted: notify the user (Rust-only requirement 3).
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
log::error!("fetch {url}: {e}");
|
log::error!("fetch {url}: {e}");
|
||||||
let _ = reply(
|
let _ = reply(ctx.sender, chat_id, reply_to, fetch_error_message(&e)).await;
|
||||||
ctx.sender,
|
|
||||||
chat_id,
|
|
||||||
reply_to,
|
|
||||||
"Failed to fetch media from this link.",
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
}
|
}
|
||||||
Ok(Some(mut fetched)) => {
|
Ok(Some(mut fetched)) => {
|
||||||
if fetched.media.is_empty() {
|
if fetched.media.is_empty() {
|
||||||
@@ -402,23 +538,28 @@ async fn url_media(ctx: &AppContext<'_>, chat_id: i64, reply_to_message_id: i64,
|
|||||||
let caption = fetched.caption_with(&format);
|
let caption = fetched.caption_with(&format);
|
||||||
// Raw render data for the link cache; the send fills in the
|
// Raw render data for the link cache; the send fills in the
|
||||||
// Telegram file ids and persists the entry.
|
// Telegram file ids and persists the entry.
|
||||||
let cache_data = fetched
|
let cache_data =
|
||||||
.render_fields()
|
fetched
|
||||||
.map(|(author, author_url, title, tags)| CachedPost {
|
.render_fields()
|
||||||
url: fetched.source_url.clone(),
|
.map(|(author, author_url, title, content, tags)| CachedPost {
|
||||||
caption: fetched.caption.clone(),
|
url: fetched.source_url.clone(),
|
||||||
title: title.to_string(),
|
caption: fetched.caption.clone(),
|
||||||
author: author.to_string(),
|
title: title.to_string(),
|
||||||
author_url: author_url.to_string(),
|
content: content.to_string(),
|
||||||
tags: tags.to_string(),
|
author: author.to_string(),
|
||||||
sensitive: fetched.sensitive,
|
author_url: author_url.to_string(),
|
||||||
media: vec![],
|
tags: tags.to_string(),
|
||||||
});
|
sensitive: fetched.sensitive,
|
||||||
|
media: vec![],
|
||||||
|
});
|
||||||
let items: Vec<MediaItemPayload> = fetched
|
let items: Vec<MediaItemPayload> = fetched
|
||||||
.media
|
.media
|
||||||
.iter()
|
.iter()
|
||||||
.map(|media| media_to_payload(media, fetched.sensitive))
|
.map(|media| media_to_payload(media, fetched.sensitive))
|
||||||
.collect();
|
.collect();
|
||||||
|
// The indicator switches to "sending photo/video" once the kinds
|
||||||
|
// are known; `items` is moved into the task below.
|
||||||
|
*hint.lock() = ActionHint::for_items(&items);
|
||||||
let task = build_send_task(
|
let task = build_send_task(
|
||||||
&chat_data,
|
&chat_data,
|
||||||
chat_id,
|
chat_id,
|
||||||
@@ -427,6 +568,7 @@ async fn url_media(ctx: &AppContext<'_>, chat_id: i64, reply_to_message_id: i64,
|
|||||||
caption,
|
caption,
|
||||||
items,
|
items,
|
||||||
cache_data,
|
cache_data,
|
||||||
|
post_send,
|
||||||
);
|
);
|
||||||
// Hand the keep-alive temp dir (ugoira / bsky remux MP4) to the
|
// Hand the keep-alive temp dir (ugoira / bsky remux MP4) to the
|
||||||
// retry registry: a queued retry runs after this function returns
|
// retry registry: a queued retry runs after this function returns
|
||||||
@@ -443,10 +585,9 @@ async fn url_media(ctx: &AppContext<'_>, chat_id: i64, reply_to_message_id: i64,
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::db;
|
use crate::ctx::test_support::TestStores;
|
||||||
use crate::link_cache::CachedMedia;
|
use crate::link_cache::CachedMedia;
|
||||||
use crate::media_sender::test_support::{MockSender, Outcome};
|
use crate::media_sender::test_support::{MockSender, Outcome};
|
||||||
use std::sync::Arc;
|
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use teloxide::{ApiError, RequestError};
|
use teloxide::{ApiError, RequestError};
|
||||||
|
|
||||||
@@ -461,6 +602,7 @@ mod tests {
|
|||||||
url: "https://x.com/u/status/1".into(),
|
url: "https://x.com/u/status/1".into(),
|
||||||
caption: "cap".into(),
|
caption: "cap".into(),
|
||||||
title: "t".into(),
|
title: "t".into(),
|
||||||
|
content: "c".into(),
|
||||||
author: "a".into(),
|
author: "a".into(),
|
||||||
author_url: "au".into(),
|
author_url: "au".into(),
|
||||||
tags: "".into(),
|
tags: "".into(),
|
||||||
@@ -474,26 +616,18 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn cache_hit_sends_file_ids_and_invalidates_on_permanent_failure() {
|
async fn cache_hit_sends_file_ids_and_invalidates_on_permanent_failure() {
|
||||||
let dir = tempfile::tempdir().unwrap();
|
let stores = TestStores::new();
|
||||||
let pool = db::open_store(dir.path().join("t.db").to_str().unwrap()).unwrap();
|
|
||||||
let chat_store = ChatStore::new(Arc::clone(&pool));
|
|
||||||
let task_queue = PersistentTaskQueue::new(Arc::clone(&pool));
|
|
||||||
let link_cache = LinkCache::new(Arc::clone(&pool));
|
|
||||||
let config = Config::load();
|
|
||||||
let sender = MockSender::scripted(
|
let sender = MockSender::scripted(
|
||||||
vec![Outcome::GroupErr, Outcome::MessageErr],
|
vec![Outcome::GroupErr, Outcome::MessageErr],
|
||||||
permanent_error,
|
permanent_error,
|
||||||
);
|
);
|
||||||
let ctx = AppContext {
|
let ctx = stores.ctx(&sender);
|
||||||
sender: &sender,
|
stores
|
||||||
chat_store: &chat_store,
|
.link_cache()
|
||||||
task_queue: &task_queue,
|
.put("twitter:1", &cached_photo_entry())
|
||||||
link_cache: &link_cache,
|
.await;
|
||||||
config: &config,
|
|
||||||
};
|
|
||||||
link_cache.put("twitter:1", &cached_photo_entry()).await;
|
|
||||||
|
|
||||||
url_media(&ctx, 1, 2, "https://x.com/u/status/1").await;
|
url_media(&ctx, 1, 2, "https://x.com/u/status/1", PostSend::FromChat).await;
|
||||||
|
|
||||||
// The cached file id went out as a group send; the permanent failure
|
// The cached file id went out as a group send; the permanent failure
|
||||||
// then triggered the fire-and-forget reply (its mock error is fine).
|
// then triggered the fire-and-forget reply (its mock error is fine).
|
||||||
@@ -503,7 +637,8 @@ mod tests {
|
|||||||
);
|
);
|
||||||
// The stale cache entry was invalidated so the next request re-fetches.
|
// The stale cache entry was invalidated so the next request re-fetches.
|
||||||
assert!(
|
assert!(
|
||||||
link_cache
|
stores
|
||||||
|
.link_cache()
|
||||||
.get("twitter:1", Duration::from_secs(3600))
|
.get("twitter:1", Duration::from_secs(3600))
|
||||||
.await
|
.await
|
||||||
.is_none()
|
.is_none()
|
||||||
@@ -512,54 +647,271 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn cache_hit_success_keeps_the_cache_entry() {
|
async fn cache_hit_success_keeps_the_cache_entry() {
|
||||||
let dir = tempfile::tempdir().unwrap();
|
let stores = TestStores::new();
|
||||||
let pool = db::open_store(dir.path().join("t.db").to_str().unwrap()).unwrap();
|
|
||||||
let chat_store = ChatStore::new(Arc::clone(&pool));
|
|
||||||
let task_queue = PersistentTaskQueue::new(Arc::clone(&pool));
|
|
||||||
let link_cache = LinkCache::new(Arc::clone(&pool));
|
|
||||||
let config = Config::load();
|
|
||||||
let sender = MockSender::scripted(vec![Outcome::GroupOk], permanent_error);
|
let sender = MockSender::scripted(vec![Outcome::GroupOk], permanent_error);
|
||||||
let ctx = AppContext {
|
let ctx = stores.ctx(&sender);
|
||||||
sender: &sender,
|
stores
|
||||||
chat_store: &chat_store,
|
.link_cache()
|
||||||
task_queue: &task_queue,
|
.put("twitter:1", &cached_photo_entry())
|
||||||
link_cache: &link_cache,
|
.await;
|
||||||
config: &config,
|
|
||||||
};
|
|
||||||
link_cache.put("twitter:1", &cached_photo_entry()).await;
|
|
||||||
|
|
||||||
url_media(&ctx, 1, 2, "https://x.com/u/status/1").await;
|
url_media(&ctx, 1, 2, "https://x.com/u/status/1", PostSend::FromChat).await;
|
||||||
|
|
||||||
assert_eq!(sender.calls(), vec!["send_chat_action", "send_media_group"]);
|
assert_eq!(sender.calls(), vec!["send_chat_action", "send_media_group"]);
|
||||||
// Success must not evict the entry.
|
// Success must not evict the entry.
|
||||||
assert!(
|
assert!(
|
||||||
link_cache
|
stores
|
||||||
|
.link_cache()
|
||||||
.get("twitter:1", Duration::from_secs(3600))
|
.get("twitter:1", Duration::from_secs(3600))
|
||||||
.await
|
.await
|
||||||
.is_some()
|
.is_some()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The caption-quote threshold matches the post's text inside the caption,
|
||||||
|
/// so a long-text cache hit is quoted and a short-text one is not.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cache_hit_quotes_a_long_text_caption() {
|
||||||
|
let mut stores = TestStores::new();
|
||||||
|
stores.config_mut().caption_quote_text_chars = 3;
|
||||||
|
let prefix = "https://x.com/u/status/1\n<a href=\"au\">a</a>: ";
|
||||||
|
|
||||||
|
for (text, expected) in [
|
||||||
|
(
|
||||||
|
"abc",
|
||||||
|
format!("{prefix}<blockquote expandable>abc</blockquote>"),
|
||||||
|
),
|
||||||
|
("ab", format!("{prefix}ab")),
|
||||||
|
] {
|
||||||
|
let sender = MockSender::scripted(vec![Outcome::GroupOk], permanent_error);
|
||||||
|
let ctx = stores.ctx(&sender);
|
||||||
|
let mut entry = cached_photo_entry();
|
||||||
|
entry.caption = format!("{prefix}{text}");
|
||||||
|
entry.title = String::new();
|
||||||
|
entry.content = text.into();
|
||||||
|
stores.link_cache().put("twitter:1", &entry).await;
|
||||||
|
|
||||||
|
url_media(&ctx, 1, 2, "https://x.com/u/status/1", PostSend::FromChat).await;
|
||||||
|
|
||||||
|
assert_eq!(sender.captions(), vec![expected], "text {text:?}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn unsupported_url_is_ignored_silently() {
|
async fn unsupported_url_is_ignored_silently() {
|
||||||
let dir = tempfile::tempdir().unwrap();
|
let stores = TestStores::new();
|
||||||
let pool = db::open_store(dir.path().join("t.db").to_str().unwrap()).unwrap();
|
|
||||||
let chat_store = ChatStore::new(Arc::clone(&pool));
|
|
||||||
let task_queue = PersistentTaskQueue::new(Arc::clone(&pool));
|
|
||||||
let link_cache = LinkCache::new(Arc::clone(&pool));
|
|
||||||
let config = Config::load();
|
|
||||||
let sender = MockSender::scripted(vec![], permanent_error);
|
let sender = MockSender::scripted(vec![], permanent_error);
|
||||||
let ctx = AppContext {
|
let ctx = stores.ctx(&sender);
|
||||||
sender: &sender,
|
|
||||||
chat_store: &chat_store,
|
|
||||||
task_queue: &task_queue,
|
|
||||||
link_cache: &link_cache,
|
|
||||||
config: &config,
|
|
||||||
};
|
|
||||||
|
|
||||||
// No cache key → the fetch dispatcher returns Ok(None) without any
|
// No cache key → the fetch dispatcher returns Ok(None) without any
|
||||||
// network; nothing is sent or replied.
|
// network; nothing is sent or replied.
|
||||||
url_media(&ctx, 1, 2, "https://example.com/not-a-post").await;
|
url_media(
|
||||||
|
&ctx,
|
||||||
|
1,
|
||||||
|
2,
|
||||||
|
"https://example.com/not-a-post",
|
||||||
|
PostSend::FromChat,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
assert_eq!(sender.calls(), vec!["send_chat_action"]);
|
assert_eq!(sender.calls(), vec!["send_chat_action"]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Send modes: the URL flow vs `/test` ─────────────────────────────
|
||||||
|
|
||||||
|
/// A chat that has both post-send actions configured.
|
||||||
|
async fn seed_post_send_settings(ctx: &AppContext<'_>) {
|
||||||
|
ctx.chat_store
|
||||||
|
.update(1, |data| {
|
||||||
|
data.forward_channel_id = Some(2);
|
||||||
|
data.edit_before_forward = true;
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn chat_settings_apply_to_the_normal_link_flow() {
|
||||||
|
let stores = TestStores::new();
|
||||||
|
let sender =
|
||||||
|
MockSender::scripted(vec![Outcome::GroupOk, Outcome::MessageOk], permanent_error);
|
||||||
|
let ctx = stores.ctx(&sender);
|
||||||
|
stores
|
||||||
|
.link_cache()
|
||||||
|
.put("twitter:1", &cached_photo_entry())
|
||||||
|
.await;
|
||||||
|
seed_post_send_settings(&ctx).await;
|
||||||
|
|
||||||
|
url_media(&ctx, 1, 2, "https://x.com/u/status/1", PostSend::FromChat).await;
|
||||||
|
|
||||||
|
// Media group, then the edit prompt (edit-before-forward wins over the
|
||||||
|
// channel forward, which only runs once the prompt is confirmed).
|
||||||
|
assert_eq!(
|
||||||
|
sender.calls(),
|
||||||
|
vec!["send_chat_action", "send_media_group", "send_message"]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_mode_sends_the_media_without_forwarding_or_editing() {
|
||||||
|
let stores = TestStores::new();
|
||||||
|
// Only the group send is scripted: any forward (copy_messages) or edit
|
||||||
|
// prompt (send_message) would panic with "unexpected outcome".
|
||||||
|
let sender = MockSender::scripted(vec![Outcome::GroupOk], permanent_error);
|
||||||
|
let ctx = stores.ctx(&sender);
|
||||||
|
stores
|
||||||
|
.link_cache()
|
||||||
|
.put("twitter:1", &cached_photo_entry())
|
||||||
|
.await;
|
||||||
|
seed_post_send_settings(&ctx).await;
|
||||||
|
|
||||||
|
url_media(&ctx, 1, 2, "https://x.com/u/status/1", PostSend::Suppressed).await;
|
||||||
|
|
||||||
|
assert_eq!(sender.calls(), vec!["send_chat_action", "send_media_group"]);
|
||||||
|
// The send is otherwise ordinary: the post stays cached.
|
||||||
|
assert!(
|
||||||
|
stores
|
||||||
|
.link_cache()
|
||||||
|
.get("twitter:1", Duration::from_secs(3600))
|
||||||
|
.await
|
||||||
|
.is_some()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn send_mode_decides_whether_chat_actions_ride_along() {
|
||||||
|
let chat = ChatData {
|
||||||
|
forward_channel_id: Some(2),
|
||||||
|
edit_before_forward: true,
|
||||||
|
..ChatData::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let with_chat = build_send_task(
|
||||||
|
&chat,
|
||||||
|
1,
|
||||||
|
2,
|
||||||
|
"https://x.com/u/status/1".into(),
|
||||||
|
"cap".into(),
|
||||||
|
vec![],
|
||||||
|
None,
|
||||||
|
PostSend::FromChat,
|
||||||
|
);
|
||||||
|
let Task::SendMediaSequence {
|
||||||
|
edit_before_forward,
|
||||||
|
forward_channel_id,
|
||||||
|
..
|
||||||
|
} = with_chat
|
||||||
|
else {
|
||||||
|
panic!("expected a media sequence task");
|
||||||
|
};
|
||||||
|
assert!(edit_before_forward);
|
||||||
|
assert_eq!(forward_channel_id, Some(2));
|
||||||
|
|
||||||
|
let suppressed = build_send_task(
|
||||||
|
&chat,
|
||||||
|
1,
|
||||||
|
2,
|
||||||
|
"https://x.com/u/status/1".into(),
|
||||||
|
"cap".into(),
|
||||||
|
vec![],
|
||||||
|
None,
|
||||||
|
PostSend::Suppressed,
|
||||||
|
);
|
||||||
|
let Task::SendMediaSequence {
|
||||||
|
edit_before_forward,
|
||||||
|
forward_channel_id,
|
||||||
|
notify_chat_id,
|
||||||
|
..
|
||||||
|
} = suppressed
|
||||||
|
else {
|
||||||
|
panic!("expected a media sequence task");
|
||||||
|
};
|
||||||
|
assert!(!edit_before_forward, "`/test` must not open an edit prompt");
|
||||||
|
assert_eq!(forward_channel_id, None, "`/test` must not forward");
|
||||||
|
// Dead-letter notification still reaches the chat that asked.
|
||||||
|
assert_eq!(notify_chat_id, Some(1));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fetch_errors_map_to_distinct_user_messages() {
|
||||||
|
use x_media::site::FetchError;
|
||||||
|
|
||||||
|
let disabled = fetch_error_message(&FetchError::Disabled { site: "pixiv" });
|
||||||
|
assert_eq!(disabled, "Pixiv support is disabled on this bot.");
|
||||||
|
assert_eq!(
|
||||||
|
fetch_error_message(&FetchError::NotFound),
|
||||||
|
"Post not found (deleted, private or unavailable)."
|
||||||
|
);
|
||||||
|
let sensitive = fetch_error_message(&FetchError::Sensitive);
|
||||||
|
assert!(sensitive.contains("TWITTER_AUTH_TOKEN"), "{sensitive}");
|
||||||
|
let blocked = fetch_error_message(&FetchError::Blocked);
|
||||||
|
assert!(blocked.contains("refused"), "{blocked}");
|
||||||
|
|
||||||
|
// Each class that has something to say must differ from the generic
|
||||||
|
// fallback — one generic sentence for everything is what this fixes.
|
||||||
|
let generic = fetch_error_message(&FetchError::TooLarge);
|
||||||
|
for text in [disabled, sensitive, blocked] {
|
||||||
|
assert_ne!(text, generic);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn action_hint_follows_the_media_kind() {
|
||||||
|
use MediaItemPayload::{Animation, Photo, Video};
|
||||||
|
|
||||||
|
let photo = || Photo {
|
||||||
|
media: "https://p/1.jpg".into(),
|
||||||
|
has_spoiler: false,
|
||||||
|
fallback_url: None,
|
||||||
|
file_id: false,
|
||||||
|
};
|
||||||
|
let video = || Video {
|
||||||
|
media: "https://v/1.mp4".into(),
|
||||||
|
has_spoiler: false,
|
||||||
|
thumbnail: None,
|
||||||
|
fallback_url: None,
|
||||||
|
file_id: false,
|
||||||
|
};
|
||||||
|
|
||||||
|
// Unknown before the fetch: the pipeline starts on "typing".
|
||||||
|
assert!(matches!(ActionHint::Typing.action(), ChatAction::Typing));
|
||||||
|
assert!(matches!(
|
||||||
|
ActionHint::for_items(&[photo()]).action(),
|
||||||
|
ChatAction::UploadPhoto
|
||||||
|
));
|
||||||
|
assert!(matches!(
|
||||||
|
ActionHint::for_items(&[
|
||||||
|
video(),
|
||||||
|
Animation {
|
||||||
|
media: "https://v/2.mp4".into(),
|
||||||
|
has_spoiler: false,
|
||||||
|
file_id: false,
|
||||||
|
}
|
||||||
|
])
|
||||||
|
.action(),
|
||||||
|
ChatAction::UploadVideo
|
||||||
|
));
|
||||||
|
// A mixed post takes the photo label: `photos_first` always leads with
|
||||||
|
// a photo, which is what Telegram shows.
|
||||||
|
assert!(matches!(
|
||||||
|
ActionHint::for_items(&[video(), photo()]).action(),
|
||||||
|
ChatAction::UploadPhoto
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn a_long_pipeline_keeps_the_chat_action_alive() {
|
||||||
|
let sender = MockSender::scripted(vec![], permanent_error);
|
||||||
|
let hint = parking_lot::Mutex::new(ActionHint::Typing);
|
||||||
|
// Three refresh windows of work: Telegram would have dropped the
|
||||||
|
// indicator twice without the keep-alive.
|
||||||
|
let pipeline = async { tokio::time::sleep(ACTION_REFRESH * 3).await };
|
||||||
|
|
||||||
|
run_with_chat_action(&sender, 1, &hint, pipeline).await;
|
||||||
|
|
||||||
|
let actions = sender
|
||||||
|
.calls()
|
||||||
|
.iter()
|
||||||
|
.filter(|call| **call == "send_chat_action")
|
||||||
|
.count();
|
||||||
|
assert_eq!(actions, 3, "expected the initial action plus two refreshes");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,7 +5,7 @@
|
|||||||
//! key]. A repeated link is then answered entirely from local state — no
|
//! key]. A repeated link is then answered entirely from local state — no
|
||||||
//! re-fetch of the source site, no re-upload — and no media file is stored
|
//! re-fetch of the source site, no re-upload — and no media file is stored
|
||||||
//! on disk (the file ids point at Telegram's servers). Entries expire after
|
//! on disk (the file ids point at Telegram's servers). Entries expire after
|
||||||
//! [`Config::link_cache_ttl`]; a stale entry is dropped lazily on read and
|
//! `Config::link_cache_ttl`; a stale entry is dropped lazily on read and
|
||||||
//! by the periodic prune in `main`.
|
//! by the periodic prune in `main`.
|
||||||
|
|
||||||
use crate::db::now_f64;
|
use crate::db::now_f64;
|
||||||
@@ -38,6 +38,10 @@ pub struct CachedPost {
|
|||||||
/// override).
|
/// override).
|
||||||
pub caption: String,
|
pub caption: String,
|
||||||
pub title: String,
|
pub title: String,
|
||||||
|
/// The post's body text. Defaulted on read: entries written before the
|
||||||
|
/// title/content split carry it inside `title`.
|
||||||
|
#[serde(default)]
|
||||||
|
pub content: String,
|
||||||
pub author: String,
|
pub author: String,
|
||||||
pub author_url: String,
|
pub author_url: String,
|
||||||
pub tags: String,
|
pub tags: String,
|
||||||
@@ -78,9 +82,15 @@ impl LinkCache {
|
|||||||
conn.execute("DELETE FROM link_cache WHERE url = ?1", params![key])?;
|
conn.execute("DELETE FROM link_cache WHERE url = ?1", params![key])?;
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
Ok(Some(serde_json::from_str::<CachedPost>(&payload).map_err(
|
match serde_json::from_str::<CachedPost>(&payload) {
|
||||||
|e| rusqlite::Error::ToSqlConversionFailure(Box::new(e)),
|
Ok(post) => Ok(Some(post)),
|
||||||
)?))
|
Err(e) => {
|
||||||
|
// Unreadable payload (e.g. an older schema): drop it
|
||||||
|
// instead of re-failing the parse on every later hit.
|
||||||
|
conn.execute("DELETE FROM link_cache WHERE url = ?1", params![key])?;
|
||||||
|
Err(rusqlite::Error::ToSqlConversionFailure(Box::new(e)))
|
||||||
|
}
|
||||||
|
}
|
||||||
})
|
})
|
||||||
.await;
|
.await;
|
||||||
match result {
|
match result {
|
||||||
@@ -176,6 +186,7 @@ mod tests {
|
|||||||
url: "https://x.com/u/status/1".into(),
|
url: "https://x.com/u/status/1".into(),
|
||||||
caption: "cap".into(),
|
caption: "cap".into(),
|
||||||
title: "t".into(),
|
title: "t".into(),
|
||||||
|
content: "c".into(),
|
||||||
author: "a".into(),
|
author: "a".into(),
|
||||||
author_url: "au".into(),
|
author_url: "au".into(),
|
||||||
tags: "".into(),
|
tags: "".into(),
|
||||||
@@ -187,6 +198,49 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A payload written before the title/content split has no `content`
|
||||||
|
/// field. It must still read back — the cache deletes what it cannot
|
||||||
|
/// parse — with its text left where it was stored (`title`) and the
|
||||||
|
/// caption it replays untouched. No migration: a self-hosted cache entry
|
||||||
|
/// lives one TTL, and moving the text would only reshuffle `/set_format`
|
||||||
|
/// placeholders until it expires.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn pre_split_entry_still_parses() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let cache = LinkCache::new(
|
||||||
|
crate::db::open_store(dir.path().join("c.db").to_str().unwrap()).unwrap(),
|
||||||
|
);
|
||||||
|
let legacy = serde_json::json!({
|
||||||
|
"url": "https://x.com/u/status/1",
|
||||||
|
"caption": "https://x.com/u/status/1\n<a href=\"au\">a</a>: old text",
|
||||||
|
"title": "old text",
|
||||||
|
"author": "a",
|
||||||
|
"author_url": "au",
|
||||||
|
"tags": "",
|
||||||
|
"sensitive": false,
|
||||||
|
"media": [{"kind": "photo", "file_id": "AgAC..."}]
|
||||||
|
});
|
||||||
|
{
|
||||||
|
let conn = rusqlite::Connection::open(dir.path().join("c.db")).unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO link_cache (url, payload, created_at) VALUES (?1, ?2, ?3)",
|
||||||
|
params!["twitter:1", legacy.to_string(), now_f64()],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
let got = cache
|
||||||
|
.get("twitter:1", Duration::from_secs(3600))
|
||||||
|
.await
|
||||||
|
.expect("a pre-split payload must not be dropped");
|
||||||
|
assert_eq!(got.title, "old text");
|
||||||
|
assert_eq!(got.content, "");
|
||||||
|
assert_eq!(
|
||||||
|
got.caption,
|
||||||
|
"https://x.com/u/status/1\n<a href=\"au\">a</a>: old text"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn put_get_roundtrip() {
|
async fn put_get_roundtrip() {
|
||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = tempfile::tempdir().unwrap();
|
||||||
@@ -228,6 +282,32 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn unreadable_entry_is_dropped_on_read() {
|
||||||
|
// A payload from an older schema must not be re-parsed on every hit:
|
||||||
|
// the row is removed and the read reports a miss.
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let db_path = dir.path().join("c.db");
|
||||||
|
let cache = LinkCache::new(crate::db::open_store(db_path.to_str().unwrap()).unwrap());
|
||||||
|
{
|
||||||
|
let conn = rusqlite::Connection::open(&db_path).unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO link_cache (url, payload, created_at) VALUES (?1, ?2, ?3)",
|
||||||
|
params!["twitter:1", "{not json", now_f64()],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
cache
|
||||||
|
.get("twitter:1", Duration::from_secs(3600))
|
||||||
|
.await
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
// Dropped, not left behind for the next hit.
|
||||||
|
assert_eq!(cache.clear(None).await, 0, "corrupted row still present");
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn remove_and_prune() {
|
async fn remove_and_prune() {
|
||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
|||||||
@@ -2,12 +2,13 @@ use dotenv::dotenv;
|
|||||||
use teloxide::dptree::endpoint;
|
use teloxide::dptree::endpoint;
|
||||||
use teloxide::prelude::*;
|
use teloxide::prelude::*;
|
||||||
use teloxide::stop::StopToken;
|
use teloxide::stop::StopToken;
|
||||||
use teloxide::types::{ChatId, InputFile, MessageId};
|
use teloxide::types::{ChatId, InlineKeyboardMarkup, InputFile, MessageId};
|
||||||
use teloxide::update_listeners::{self, UpdateListener, webhooks};
|
use teloxide::update_listeners::{self, UpdateListener, webhooks};
|
||||||
use tokio::sync::watch;
|
use tokio::sync::watch;
|
||||||
use x_media::site;
|
use x_media::site;
|
||||||
|
|
||||||
mod config;
|
mod config;
|
||||||
|
mod ctx;
|
||||||
mod db;
|
mod db;
|
||||||
mod handlers;
|
mod handlers;
|
||||||
mod link_cache;
|
mod link_cache;
|
||||||
@@ -18,6 +19,7 @@ mod rate_limit;
|
|||||||
mod send;
|
mod send;
|
||||||
mod state;
|
mod state;
|
||||||
|
|
||||||
|
use ctx::CONTEXT;
|
||||||
use handlers::{CHAT_STORE, CONFIG, LINK_CACHE, TASK_QUEUE};
|
use handlers::{CHAT_STORE, CONFIG, LINK_CACHE, TASK_QUEUE};
|
||||||
|
|
||||||
/// Docker `stop` / `compose down` delivers SIGTERM, which teloxide's ctrlc
|
/// Docker `stop` / `compose down` delivers SIGTERM, which teloxide's ctrlc
|
||||||
@@ -61,9 +63,13 @@ async fn main() {
|
|||||||
);
|
);
|
||||||
|
|
||||||
// Queue worker: handles typed tasks, dead-letters failed sends to the
|
// Queue worker: handles typed tasks, dead-letters failed sends to the
|
||||||
// task's chat.
|
// task's chat. Both closures use the shared context (the queue requires
|
||||||
|
// 'static handlers, and the statics are process-wide anyway).
|
||||||
TASK_QUEUE
|
TASK_QUEUE
|
||||||
.start(send::handle_task, send::dead_letter_notify)
|
.start(
|
||||||
|
|payload| send::handle_task(&CONTEXT, payload),
|
||||||
|
|payload, message| send::dead_letter_notify(&CONTEXT, payload, message),
|
||||||
|
)
|
||||||
.await;
|
.await;
|
||||||
log::info!("task queue worker started");
|
log::info!("task queue worker started");
|
||||||
|
|
||||||
@@ -108,14 +114,24 @@ async fn main() {
|
|||||||
if pruned > 0 {
|
if pruned > 0 {
|
||||||
log::info!("link cache: pruned {pruned} expired entr(ies)");
|
log::info!("link cache: pruned {pruned} expired entr(ies)");
|
||||||
}
|
}
|
||||||
|
let idle_limiters = crate::rate_limit::prune_idle();
|
||||||
|
if idle_limiters > 0 {
|
||||||
|
log::debug!("rate limiter: dropped {idle_limiters} idle bucket(s)");
|
||||||
|
}
|
||||||
for (chat_id, prompt_message_id) in removed {
|
for (chat_id, prompt_message_id) in removed {
|
||||||
// If the prompt was already deleted, this fails with a
|
// Rewritten in place, not announced: the sweep is a
|
||||||
// 400 "message to edit not found" — log and ignore.
|
// background timer, and a fresh message would wake the chat
|
||||||
|
// up to a full TTL later about a prompt the user already
|
||||||
|
// walked away from. The edit drops the buttons too. If the
|
||||||
|
// prompt was already deleted this fails with a 400
|
||||||
|
// "message to edit not found" — log and ignore.
|
||||||
if let Err(e) = bot
|
if let Err(e) = bot
|
||||||
.edit_message_reply_markup(
|
.edit_message_text(
|
||||||
ChatId(chat_id),
|
ChatId(chat_id),
|
||||||
MessageId(prompt_message_id as i32),
|
MessageId(prompt_message_id as i32),
|
||||||
|
send::EDIT_PROMPT_EXPIRED_TEXT,
|
||||||
)
|
)
|
||||||
|
.reply_markup(InlineKeyboardMarkup::default())
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
log::info!("edit-expiry sweep: prompt message gone: {e}");
|
log::info!("edit-expiry sweep: prompt message gone: {e}");
|
||||||
|
|||||||
@@ -8,8 +8,8 @@ use teloxide::RequestError;
|
|||||||
use teloxide::prelude::Requester;
|
use teloxide::prelude::Requester;
|
||||||
use teloxide::prelude::*;
|
use teloxide::prelude::*;
|
||||||
use teloxide::types::{
|
use teloxide::types::{
|
||||||
ChatAction, ChatId, InlineKeyboardMarkup, InputFile, InputMedia, Message, MessageId, ParseMode,
|
CallbackQueryId, ChatAction, ChatId, InlineKeyboardMarkup, InputFile, InputMedia, Message,
|
||||||
ReplyParameters,
|
MessageId, ParseMode, ReplyParameters,
|
||||||
};
|
};
|
||||||
|
|
||||||
/// Boxed, `Send` future returned by a [`MediaSender`] method (`async fn` in
|
/// Boxed, `Send` future returned by a [`MediaSender`] method (`async fn` in
|
||||||
@@ -48,14 +48,42 @@ pub trait MediaSender: Send + Sync {
|
|||||||
) -> BoxFuture<'_, Result<Vec<MessageId>, RequestError>>;
|
) -> BoxFuture<'_, Result<Vec<MessageId>, RequestError>>;
|
||||||
|
|
||||||
/// Sends a plain text message, optionally replying to `reply_to` and
|
/// Sends a plain text message, optionally replying to `reply_to` and
|
||||||
/// attaching `reply_markup`.
|
/// attaching `reply_markup`. Returns the sent message's id: the bot only
|
||||||
|
/// ever needs that (the edit-before-forward prompt's record is keyed by
|
||||||
|
/// it), and returning the whole `Message` would force every test mock to
|
||||||
|
/// construct one.
|
||||||
fn send_message(
|
fn send_message(
|
||||||
&self,
|
&self,
|
||||||
chat_id: ChatId,
|
chat_id: ChatId,
|
||||||
text: String,
|
text: String,
|
||||||
reply_to: Option<MessageId>,
|
reply_to: Option<MessageId>,
|
||||||
reply_markup: Option<InlineKeyboardMarkup>,
|
reply_markup: Option<InlineKeyboardMarkup>,
|
||||||
) -> BoxFuture<'_, Result<Message, RequestError>>;
|
) -> BoxFuture<'_, Result<i64, RequestError>>;
|
||||||
|
|
||||||
|
/// Answers a callback query, optionally with a toast `text` shown to the
|
||||||
|
/// user who pressed the button.
|
||||||
|
fn answer_callback_query(
|
||||||
|
&self,
|
||||||
|
id: CallbackQueryId,
|
||||||
|
text: Option<String>,
|
||||||
|
) -> BoxFuture<'_, Result<(), RequestError>>;
|
||||||
|
|
||||||
|
/// Rewrites a message's caption, always with HTML parse mode (every caller
|
||||||
|
/// in this bot renders escaped HTML: templates and edit-before-forward
|
||||||
|
/// links).
|
||||||
|
fn edit_message_caption(
|
||||||
|
&self,
|
||||||
|
chat_id: ChatId,
|
||||||
|
message_id: MessageId,
|
||||||
|
caption: String,
|
||||||
|
) -> BoxFuture<'_, Result<(), RequestError>>;
|
||||||
|
|
||||||
|
/// Deletes a message (the edit-before-forward prompt after a forward).
|
||||||
|
fn delete_message(
|
||||||
|
&self,
|
||||||
|
chat_id: ChatId,
|
||||||
|
message_id: MessageId,
|
||||||
|
) -> BoxFuture<'_, Result<(), RequestError>>;
|
||||||
|
|
||||||
/// Sets the chat's "typing / uploading …" indicator (cosmetic).
|
/// Sets the chat's "typing / uploading …" indicator (cosmetic).
|
||||||
fn send_chat_action(
|
fn send_chat_action(
|
||||||
@@ -129,7 +157,7 @@ impl MediaSender for Bot {
|
|||||||
text: String,
|
text: String,
|
||||||
reply_to: Option<MessageId>,
|
reply_to: Option<MessageId>,
|
||||||
reply_markup: Option<InlineKeyboardMarkup>,
|
reply_markup: Option<InlineKeyboardMarkup>,
|
||||||
) -> BoxFuture<'_, Result<Message, RequestError>> {
|
) -> BoxFuture<'_, Result<i64, RequestError>> {
|
||||||
Box::pin(async move {
|
Box::pin(async move {
|
||||||
let mut request = <Bot as Requester>::send_message(self, chat_id, text);
|
let mut request = <Bot as Requester>::send_message(self, chat_id, text);
|
||||||
if let Some(reply_to) = reply_to {
|
if let Some(reply_to) = reply_to {
|
||||||
@@ -139,7 +167,48 @@ impl MediaSender for Bot {
|
|||||||
if let Some(markup) = reply_markup {
|
if let Some(markup) = reply_markup {
|
||||||
request = request.reply_markup(markup);
|
request = request.reply_markup(markup);
|
||||||
}
|
}
|
||||||
request.await
|
request.await.map(|message| message.id.0 as i64)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn answer_callback_query(
|
||||||
|
&self,
|
||||||
|
id: CallbackQueryId,
|
||||||
|
text: Option<String>,
|
||||||
|
) -> BoxFuture<'_, Result<(), RequestError>> {
|
||||||
|
Box::pin(async move {
|
||||||
|
let mut request = <Bot as Requester>::answer_callback_query(self, id);
|
||||||
|
if let Some(text) = text {
|
||||||
|
request = request.text(text);
|
||||||
|
}
|
||||||
|
request.await.map(|_| ())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn edit_message_caption(
|
||||||
|
&self,
|
||||||
|
chat_id: ChatId,
|
||||||
|
message_id: MessageId,
|
||||||
|
caption: String,
|
||||||
|
) -> BoxFuture<'_, Result<(), RequestError>> {
|
||||||
|
Box::pin(async move {
|
||||||
|
<Bot as Requester>::edit_message_caption(self, chat_id, message_id)
|
||||||
|
.caption(caption)
|
||||||
|
.parse_mode(ParseMode::Html)
|
||||||
|
.await
|
||||||
|
.map(|_| ())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn delete_message(
|
||||||
|
&self,
|
||||||
|
chat_id: ChatId,
|
||||||
|
message_id: MessageId,
|
||||||
|
) -> BoxFuture<'_, Result<(), RequestError>> {
|
||||||
|
Box::pin(async move {
|
||||||
|
<Bot as Requester>::delete_message(self, chat_id, message_id)
|
||||||
|
.await
|
||||||
|
.map(|_| ())
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -162,7 +231,7 @@ impl MediaSender for Bot {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) mod test_support {
|
pub(crate) mod test_support {
|
||||||
use super::*;
|
use super::*;
|
||||||
use std::sync::Mutex;
|
use parking_lot::Mutex;
|
||||||
|
|
||||||
/// One scripted outcome, consumed front-to-back; the last entry repeats
|
/// One scripted outcome, consumed front-to-back; the last entry repeats
|
||||||
/// for further calls of the same method kind.
|
/// for further calls of the same method kind.
|
||||||
@@ -176,19 +245,30 @@ pub(crate) mod test_support {
|
|||||||
/// An error from `send_message` (replies are fire-and-forget, so an
|
/// An error from `send_message` (replies are fire-and-forget, so an
|
||||||
/// error is fine for tests).
|
/// error is fine for tests).
|
||||||
MessageErr,
|
MessageErr,
|
||||||
|
/// A successful `send_message`, returning message id [`MockSender::SENT_ID`].
|
||||||
|
MessageOk,
|
||||||
|
EditOk,
|
||||||
|
EditErr,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Replays a script and records the method names that were called.
|
/// Replays a script and records what was sent, so tests can assert the
|
||||||
|
/// user-visible text a path produced.
|
||||||
pub(crate) struct MockSender {
|
pub(crate) struct MockSender {
|
||||||
script: Mutex<Vec<Outcome>>,
|
script: Mutex<Vec<Outcome>>,
|
||||||
cursor: Mutex<usize>,
|
cursor: Mutex<usize>,
|
||||||
calls: Mutex<Vec<&'static str>>,
|
calls: Mutex<Vec<&'static str>>,
|
||||||
|
messages: Mutex<Vec<String>>,
|
||||||
|
captions: Mutex<Vec<String>>,
|
||||||
|
answers: Mutex<Vec<Option<String>>>,
|
||||||
/// Builds the error every `*Err` outcome returns (RequestError is not
|
/// Builds the error every `*Err` outcome returns (RequestError is not
|
||||||
/// cloneable, so the factory recreates it per call).
|
/// cloneable, so the factory recreates it per call).
|
||||||
error: Box<dyn Fn() -> RequestError + Send + Sync>,
|
error: Box<dyn Fn() -> RequestError + Send + Sync>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl MockSender {
|
impl MockSender {
|
||||||
|
/// The message id a successful `send_message` reports.
|
||||||
|
pub(crate) const SENT_ID: i64 = 1;
|
||||||
|
|
||||||
pub(crate) fn scripted(
|
pub(crate) fn scripted(
|
||||||
script: Vec<Outcome>,
|
script: Vec<Outcome>,
|
||||||
error: impl Fn() -> RequestError + Send + Sync + 'static,
|
error: impl Fn() -> RequestError + Send + Sync + 'static,
|
||||||
@@ -197,6 +277,9 @@ pub(crate) mod test_support {
|
|||||||
script: Mutex::new(script),
|
script: Mutex::new(script),
|
||||||
cursor: Mutex::new(0),
|
cursor: Mutex::new(0),
|
||||||
calls: Mutex::new(Vec::new()),
|
calls: Mutex::new(Vec::new()),
|
||||||
|
messages: Mutex::new(Vec::new()),
|
||||||
|
captions: Mutex::new(Vec::new()),
|
||||||
|
answers: Mutex::new(Vec::new()),
|
||||||
error: Box::new(error),
|
error: Box::new(error),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -204,13 +287,28 @@ pub(crate) mod test_support {
|
|||||||
/// Method names in call order (e.g. `["send_media_group",
|
/// Method names in call order (e.g. `["send_media_group",
|
||||||
/// "send_media_group"]` proves the fallback re-sent).
|
/// "send_media_group"]` proves the fallback re-sent).
|
||||||
pub(crate) fn calls(&self) -> Vec<&'static str> {
|
pub(crate) fn calls(&self) -> Vec<&'static str> {
|
||||||
self.calls.lock().unwrap().clone()
|
self.calls.lock().clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Texts of the plain messages sent, in order.
|
||||||
|
pub(crate) fn messages(&self) -> Vec<String> {
|
||||||
|
self.messages.lock().clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Captions passed to `edit_message_caption`, in order.
|
||||||
|
pub(crate) fn captions(&self) -> Vec<String> {
|
||||||
|
self.captions.lock().clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Toast texts of the answered callback queries, in order.
|
||||||
|
pub(crate) fn answers(&self) -> Vec<Option<String>> {
|
||||||
|
self.answers.lock().clone()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn next(&self, kind: &'static str) -> Outcome {
|
fn next(&self, kind: &'static str) -> Outcome {
|
||||||
self.calls.lock().unwrap().push(kind);
|
self.calls.lock().push(kind);
|
||||||
let script = self.script.lock().unwrap();
|
let script = self.script.lock();
|
||||||
let mut cursor = self.cursor.lock().unwrap();
|
let mut cursor = self.cursor.lock();
|
||||||
if script.is_empty() {
|
if script.is_empty() {
|
||||||
panic!("mock script exhausted: {kind}");
|
panic!("mock script exhausted: {kind}");
|
||||||
}
|
}
|
||||||
@@ -229,9 +327,20 @@ pub(crate) mod test_support {
|
|||||||
&self,
|
&self,
|
||||||
_chat_id: ChatId,
|
_chat_id: ChatId,
|
||||||
_reply_to: MessageId,
|
_reply_to: MessageId,
|
||||||
_items: Vec<InputMedia>,
|
items: Vec<InputMedia>,
|
||||||
) -> BoxFuture<'_, Result<Vec<Message>, RequestError>> {
|
) -> BoxFuture<'_, Result<Vec<Message>, RequestError>> {
|
||||||
Box::pin(async move {
|
Box::pin(async move {
|
||||||
|
// Record the captions exactly as Telegram receives them (only
|
||||||
|
// the first item of a group carries one), so tests can assert
|
||||||
|
// what a recipient sees.
|
||||||
|
self.captions
|
||||||
|
.lock()
|
||||||
|
.extend(items.iter().filter_map(|item| match item {
|
||||||
|
InputMedia::Photo(photo) => photo.caption.clone(),
|
||||||
|
InputMedia::Video(video) => video.caption.clone(),
|
||||||
|
InputMedia::Animation(animation) => animation.caption.clone(),
|
||||||
|
_ => None,
|
||||||
|
}));
|
||||||
match self.next("send_media_group") {
|
match self.next("send_media_group") {
|
||||||
Outcome::GroupOk => Ok(Vec::new()),
|
Outcome::GroupOk => Ok(Vec::new()),
|
||||||
Outcome::GroupErr => Err(self.error()),
|
Outcome::GroupErr => Err(self.error()),
|
||||||
@@ -274,25 +383,69 @@ pub(crate) mod test_support {
|
|||||||
fn send_message(
|
fn send_message(
|
||||||
&self,
|
&self,
|
||||||
_chat_id: ChatId,
|
_chat_id: ChatId,
|
||||||
_text: String,
|
text: String,
|
||||||
_reply_to: Option<MessageId>,
|
_reply_to: Option<MessageId>,
|
||||||
_reply_markup: Option<InlineKeyboardMarkup>,
|
_reply_markup: Option<InlineKeyboardMarkup>,
|
||||||
) -> BoxFuture<'_, Result<Message, RequestError>> {
|
) -> BoxFuture<'_, Result<i64, RequestError>> {
|
||||||
Box::pin(async move {
|
Box::pin(async move {
|
||||||
|
self.messages.lock().push(text);
|
||||||
match self.next("send_message") {
|
match self.next("send_message") {
|
||||||
|
Outcome::MessageOk => Ok(MockSender::SENT_ID),
|
||||||
Outcome::MessageErr => Err(self.error()),
|
Outcome::MessageErr => Err(self.error()),
|
||||||
other => panic!("unexpected outcome {other:?} for send_message"),
|
other => panic!("unexpected outcome {other:?} for send_message"),
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn answer_callback_query(
|
||||||
|
&self,
|
||||||
|
_id: CallbackQueryId,
|
||||||
|
text: Option<String>,
|
||||||
|
) -> BoxFuture<'_, Result<(), RequestError>> {
|
||||||
|
// Always succeeds: the toast is cosmetic, so the script stays
|
||||||
|
// focused on the outcomes a test cares about.
|
||||||
|
Box::pin(async move {
|
||||||
|
self.calls.lock().push("answer_callback_query");
|
||||||
|
self.answers.lock().push(text);
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn edit_message_caption(
|
||||||
|
&self,
|
||||||
|
_chat_id: ChatId,
|
||||||
|
_message_id: MessageId,
|
||||||
|
caption: String,
|
||||||
|
) -> BoxFuture<'_, Result<(), RequestError>> {
|
||||||
|
Box::pin(async move {
|
||||||
|
self.captions.lock().push(caption);
|
||||||
|
match self.next("edit_message_caption") {
|
||||||
|
Outcome::EditOk => Ok(()),
|
||||||
|
Outcome::EditErr => Err(self.error()),
|
||||||
|
other => panic!("unexpected outcome {other:?} for edit_message_caption"),
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn delete_message(
|
||||||
|
&self,
|
||||||
|
_chat_id: ChatId,
|
||||||
|
_message_id: MessageId,
|
||||||
|
) -> BoxFuture<'_, Result<(), RequestError>> {
|
||||||
|
// Deletion is fire-and-forget in every caller; always succeeds.
|
||||||
|
Box::pin(async move {
|
||||||
|
self.calls.lock().push("delete_message");
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn send_chat_action(
|
fn send_chat_action(
|
||||||
&self,
|
&self,
|
||||||
_chat_id: ChatId,
|
_chat_id: ChatId,
|
||||||
_action: ChatAction,
|
_action: ChatAction,
|
||||||
) -> BoxFuture<'_, Result<(), RequestError>> {
|
) -> BoxFuture<'_, Result<(), RequestError>> {
|
||||||
Box::pin(async move {
|
Box::pin(async move {
|
||||||
self.calls.lock().unwrap().push("send_chat_action");
|
self.calls.lock().push("send_chat_action");
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -41,7 +41,13 @@ type DeadLetter = dyn Fn(Value, String) -> BoxFuture<'static, ()> + Send + Sync;
|
|||||||
|
|
||||||
pub struct PersistentTaskQueue {
|
pub struct PersistentTaskQueue {
|
||||||
pool: std::sync::Arc<crate::db::DbPool>,
|
pool: std::sync::Arc<crate::db::DbPool>,
|
||||||
|
/// Wakes the workers when a row becomes leasable. `notify_one` stores a
|
||||||
|
/// permit, so nothing else may share it: a waiter that is not a worker
|
||||||
|
/// (the sweep) can consume the permit and leave the due row pending until
|
||||||
|
/// the next enqueue.
|
||||||
notify: Arc<Notify>,
|
notify: Arc<Notify>,
|
||||||
|
/// Wakes the lease-expiry sweep; `stop` is the only producer.
|
||||||
|
sweep_notify: Arc<Notify>,
|
||||||
stop: Arc<AtomicBool>,
|
stop: Arc<AtomicBool>,
|
||||||
worker: Mutex<Vec<JoinHandle<()>>>,
|
worker: Mutex<Vec<JoinHandle<()>>>,
|
||||||
counter: AtomicU64,
|
counter: AtomicU64,
|
||||||
@@ -88,6 +94,7 @@ impl PersistentTaskQueue {
|
|||||||
Self {
|
Self {
|
||||||
pool,
|
pool,
|
||||||
notify: Arc::new(Notify::new()),
|
notify: Arc::new(Notify::new()),
|
||||||
|
sweep_notify: Arc::new(Notify::new()),
|
||||||
stop: Arc::new(AtomicBool::new(false)),
|
stop: Arc::new(AtomicBool::new(false)),
|
||||||
worker: Mutex::new(Vec::new()),
|
worker: Mutex::new(Vec::new()),
|
||||||
counter: AtomicU64::new(0),
|
counter: AtomicU64::new(0),
|
||||||
@@ -119,12 +126,12 @@ impl PersistentTaskQueue {
|
|||||||
handles.push(tokio::spawn(worker.run_loop_supervised()));
|
handles.push(tokio::spawn(worker.run_loop_supervised()));
|
||||||
}
|
}
|
||||||
// Periodic lease-expiry sweep: recovers rows a crashed/panicked
|
// Periodic lease-expiry sweep: recovers rows a crashed/panicked
|
||||||
// worker left `in_progress` (the lock TTL bounds the wait). Woken by
|
// worker left `in_progress` (the lock TTL bounds the wait). Its own
|
||||||
// the same notify as the workers, so enqueue and stop interrupt the
|
// notify (not the workers'): sharing that one let this task consume a
|
||||||
// sleep; the first interval tick fires immediately (harmless extra
|
// `notify_one` permit meant for a worker, which then slept through a
|
||||||
// recovery at startup).
|
// due row until some later event. Only `stop` wakes it.
|
||||||
let sweep_pool = std::sync::Arc::clone(&self.pool);
|
let sweep_pool = std::sync::Arc::clone(&self.pool);
|
||||||
let sweep_notify = Arc::clone(&self.notify);
|
let sweep_notify = Arc::clone(&self.sweep_notify);
|
||||||
let sweep_stop = Arc::clone(&self.stop);
|
let sweep_stop = Arc::clone(&self.stop);
|
||||||
handles.push(tokio::spawn(async move {
|
handles.push(tokio::spawn(async move {
|
||||||
let mut interval = tokio::time::interval(Duration::from_secs(30));
|
let mut interval = tokio::time::interval(Duration::from_secs(30));
|
||||||
@@ -150,6 +157,7 @@ impl PersistentTaskQueue {
|
|||||||
pub async fn stop(&self) {
|
pub async fn stop(&self) {
|
||||||
self.stop.store(true, Ordering::Relaxed);
|
self.stop.store(true, Ordering::Relaxed);
|
||||||
self.notify.notify_waiters();
|
self.notify.notify_waiters();
|
||||||
|
self.sweep_notify.notify_waiters();
|
||||||
let handles = std::mem::take(&mut *self.worker.lock());
|
let handles = std::mem::take(&mut *self.worker.lock());
|
||||||
for handle in handles {
|
for handle in handles {
|
||||||
let _ = handle.await;
|
let _ = handle.await;
|
||||||
@@ -334,8 +342,16 @@ impl QueueWorker {
|
|||||||
payload,
|
payload,
|
||||||
}) => {
|
}) => {
|
||||||
if row.attempts as u32 >= MAX_RETRIES {
|
if row.attempts as u32 >= MAX_RETRIES {
|
||||||
let message = format!("task failed after {MAX_RETRIES} retries");
|
// The queue keeps only the payload, not the last error, so
|
||||||
log::error!("dead-lettering {}: {message}", row.id);
|
// the cause of an exhausted retry is just that: exhausted.
|
||||||
|
// (The dead-letter message is read by the user, so it must
|
||||||
|
// not restate its own wrapper — see `failure_text`.)
|
||||||
|
let message = "retries exhausted".to_string();
|
||||||
|
log::error!(
|
||||||
|
"dead-lettering {}: {message} after {} attempt(s)",
|
||||||
|
row.id,
|
||||||
|
row.attempts + 1
|
||||||
|
);
|
||||||
self.delete_row(&row.id).await;
|
self.delete_row(&row.id).await;
|
||||||
(self.dead_letter)(payload, message).await;
|
(self.dead_letter)(payload, message).await;
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -48,6 +48,19 @@ impl TokenBucket {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Applies the elapsed refill to `state`. Shared by [`Self::acquire`] and
|
||||||
|
/// the idle check so the two cannot drift apart.
|
||||||
|
fn refill(&self, state: &mut State) {
|
||||||
|
let now = tokio::time::Instant::now();
|
||||||
|
let elapsed = now
|
||||||
|
.saturating_duration_since(state.last_refill)
|
||||||
|
.as_secs_f64();
|
||||||
|
// Refill up to the capacity; a debt (negative balance) is repaid
|
||||||
|
// before any surplus accumulates.
|
||||||
|
state.tokens = (state.tokens + elapsed * self.refill_per_sec).min(self.capacity);
|
||||||
|
state.last_refill = now;
|
||||||
|
}
|
||||||
|
|
||||||
/// Waits until `n` tokens are available, consuming them. The wait is
|
/// Waits until `n` tokens are available, consuming them. The wait is
|
||||||
/// bounded: the deficit is committed as debt and repaid over time, so a
|
/// bounded: the deficit is committed as debt and repaid over time, so a
|
||||||
/// large acquire returns once its share of the refill budget has passed.
|
/// large acquire returns once its share of the refill budget has passed.
|
||||||
@@ -57,14 +70,7 @@ impl TokenBucket {
|
|||||||
// would make the future !Send).
|
// would make the future !Send).
|
||||||
let wait = {
|
let wait = {
|
||||||
let mut state = self.state.lock();
|
let mut state = self.state.lock();
|
||||||
let now = tokio::time::Instant::now();
|
self.refill(&mut state);
|
||||||
let elapsed = now
|
|
||||||
.saturating_duration_since(state.last_refill)
|
|
||||||
.as_secs_f64();
|
|
||||||
// Refill up to the capacity; a debt (negative balance) is repaid
|
|
||||||
// before any surplus accumulates.
|
|
||||||
state.tokens = (state.tokens + elapsed * self.refill_per_sec).min(self.capacity);
|
|
||||||
state.last_refill = now;
|
|
||||||
if state.tokens >= n {
|
if state.tokens >= n {
|
||||||
state.tokens -= n;
|
state.tokens -= n;
|
||||||
return;
|
return;
|
||||||
@@ -77,6 +83,14 @@ impl TokenBucket {
|
|||||||
};
|
};
|
||||||
tokio::time::sleep(Duration::from_secs_f64(wait)).await;
|
tokio::time::sleep(Duration::from_secs_f64(wait)).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// True when the bucket has refilled to capacity: no debt outstanding, so
|
||||||
|
/// the chat has not sent anything recently.
|
||||||
|
fn is_idle(&self) -> bool {
|
||||||
|
let mut state = self.state.lock();
|
||||||
|
self.refill(&mut state);
|
||||||
|
state.tokens >= self.capacity
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// One limiter per chat, created on first use. Per-chat so one chat's burst
|
/// One limiter per chat, created on first use. Per-chat so one chat's burst
|
||||||
@@ -93,6 +107,18 @@ pub fn limiter_for(chat_id: i64) -> Arc<TokenBucket> {
|
|||||||
.clone()
|
.clone()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Drops limiters that are idle (refilled to capacity, so the chat has not
|
||||||
|
/// sent recently) and are not still held by an in-flight sender. The map
|
||||||
|
/// would otherwise keep one bucket per chat that ever sent media, forever.
|
||||||
|
/// Called from the periodic sweep; returns how many were dropped.
|
||||||
|
pub fn prune_idle() -> usize {
|
||||||
|
let mut limiters = LIMITERS.lock();
|
||||||
|
let before = limiters.len();
|
||||||
|
// Lock order map → bucket, the only order taken anywhere.
|
||||||
|
limiters.retain(|_, bucket| Arc::strong_count(bucket) > 1 || !bucket.is_idle());
|
||||||
|
before - limiters.len()
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
@@ -133,4 +159,26 @@ mod tests {
|
|||||||
start.elapsed()
|
start.elapsed()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn prune_idle_drops_full_unheld_buckets_only() {
|
||||||
|
// Held by this task: kept even at full capacity, a sender has it.
|
||||||
|
let held = limiter_for(9_001);
|
||||||
|
assert!(held.is_idle(), "a fresh bucket is full");
|
||||||
|
// Only the map holds this one and it is full → dropped.
|
||||||
|
limiter_for(9_002);
|
||||||
|
// Mid-debt (an acquire larger than the capacity): kept.
|
||||||
|
{
|
||||||
|
let bucket = Arc::new(TokenBucket::new(CAPACITY, REFILL_PER_SEC));
|
||||||
|
bucket.state.lock().tokens = -1.0;
|
||||||
|
LIMITERS.lock().insert(9_003, bucket);
|
||||||
|
}
|
||||||
|
|
||||||
|
assert!(prune_idle() >= 1);
|
||||||
|
|
||||||
|
let limiters = LIMITERS.lock();
|
||||||
|
assert!(limiters.contains_key(&9_001), "held bucket pruned");
|
||||||
|
assert!(!limiters.contains_key(&9_002), "idle unheld bucket kept");
|
||||||
|
assert!(limiters.contains_key(&9_003), "indebted bucket pruned");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,128 @@
|
|||||||
|
//! Payload → Telegram input types: `InputFile` selection (cached file id /
|
||||||
|
//! URL / local path), the per-kind `InputMedia` builders and the media-group
|
||||||
|
//! assembly with its caption rule.
|
||||||
|
|
||||||
|
use super::MediaItemPayload;
|
||||||
|
use teloxide::types::{
|
||||||
|
InputFile, InputMedia, InputMediaAnimation, InputMediaPhoto, InputMediaVideo, ParseMode,
|
||||||
|
};
|
||||||
|
|
||||||
|
fn parse_media_url(s: &str) -> Result<url::Url, String> {
|
||||||
|
url::Url::parse(s).map_err(|e| format!("invalid media URL: {e}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn item_url(item: &MediaItemPayload) -> &str {
|
||||||
|
match item {
|
||||||
|
MediaItemPayload::Photo { media, .. }
|
||||||
|
| MediaItemPayload::Video { media, .. }
|
||||||
|
| MediaItemPayload::Animation { media, .. } => media,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Remote http(s) URLs are handed to Telegram to fetch; everything else
|
||||||
|
/// (e.g. a locally encoded ugoira MP4) is uploaded directly.
|
||||||
|
pub(super) fn input_file_for(media: &str) -> Result<InputFile, String> {
|
||||||
|
if media.starts_with("http://") || media.starts_with("https://") {
|
||||||
|
Ok(InputFile::url(parse_media_url(media)?))
|
||||||
|
} else if !std::path::Path::new(media).exists() {
|
||||||
|
// A retried task may reference a temp file the original send's
|
||||||
|
// TempDir already cleaned up; fail fast and permanent instead of
|
||||||
|
// burning retries on a file that can never come back.
|
||||||
|
Err(format!("local media file missing: {media}"))
|
||||||
|
} else {
|
||||||
|
Ok(InputFile::file(media))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MediaItemPayload {
|
||||||
|
/// The input for a send: a cached file id goes out as `InputFile::file_id`
|
||||||
|
/// (no fetch, no upload), URLs go to Telegram, anything else is a local
|
||||||
|
/// path (transient upload fallback).
|
||||||
|
fn input_file(&self) -> Result<InputFile, String> {
|
||||||
|
match self {
|
||||||
|
MediaItemPayload::Photo {
|
||||||
|
media,
|
||||||
|
file_id: true,
|
||||||
|
..
|
||||||
|
}
|
||||||
|
| MediaItemPayload::Video {
|
||||||
|
media,
|
||||||
|
file_id: true,
|
||||||
|
..
|
||||||
|
}
|
||||||
|
| MediaItemPayload::Animation {
|
||||||
|
media,
|
||||||
|
file_id: true,
|
||||||
|
..
|
||||||
|
} => Ok(InputFile::file_id(media.clone().into())),
|
||||||
|
_ => input_file_for(item_url(self)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn photo_media(file: InputFile, caption: Option<&str>, spoiler: bool) -> InputMedia {
|
||||||
|
let mut photo = InputMediaPhoto::new(file).parse_mode(ParseMode::Html);
|
||||||
|
if let Some(caption) = caption {
|
||||||
|
photo = photo.caption(caption);
|
||||||
|
}
|
||||||
|
if spoiler {
|
||||||
|
photo = photo.spoiler();
|
||||||
|
}
|
||||||
|
InputMedia::Photo(photo)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn video_media(file: InputFile, caption: Option<&str>, spoiler: bool) -> InputMedia {
|
||||||
|
let mut video = InputMediaVideo::new(file).parse_mode(ParseMode::Html);
|
||||||
|
if let Some(caption) = caption {
|
||||||
|
video = video.caption(caption);
|
||||||
|
}
|
||||||
|
if spoiler {
|
||||||
|
video = video.spoiler();
|
||||||
|
}
|
||||||
|
InputMedia::Video(video)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn animation_media(file: InputFile, caption: Option<&str>, spoiler: bool) -> InputMedia {
|
||||||
|
let mut animation = InputMediaAnimation::new(file).parse_mode(ParseMode::Html);
|
||||||
|
if let Some(caption) = caption {
|
||||||
|
animation = animation.caption(caption);
|
||||||
|
}
|
||||||
|
if spoiler {
|
||||||
|
animation = animation.spoiler();
|
||||||
|
}
|
||||||
|
InputMedia::Animation(animation)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Builds a media group from payloads; only the first item of the batch gets
|
||||||
|
/// the caption (Telegram rejects captions on later items).
|
||||||
|
pub(super) fn build_media_group(
|
||||||
|
batch: &[MediaItemPayload],
|
||||||
|
caption: Option<&str>,
|
||||||
|
) -> Result<Vec<InputMedia>, String> {
|
||||||
|
batch
|
||||||
|
.iter()
|
||||||
|
.enumerate()
|
||||||
|
.map(|(i, item)| {
|
||||||
|
let item_caption = if i == 0 { caption } else { None };
|
||||||
|
Ok(match item {
|
||||||
|
MediaItemPayload::Photo { has_spoiler, .. } => {
|
||||||
|
photo_media(item.input_file()?, item_caption, *has_spoiler)
|
||||||
|
}
|
||||||
|
MediaItemPayload::Video {
|
||||||
|
has_spoiler,
|
||||||
|
thumbnail,
|
||||||
|
..
|
||||||
|
} => {
|
||||||
|
let mut video = video_media(item.input_file()?, item_caption, *has_spoiler);
|
||||||
|
if let (Some(thumb), InputMedia::Video(v)) = (thumbnail, &mut video) {
|
||||||
|
*v = v.clone().thumbnail(input_file_for(thumb)?);
|
||||||
|
}
|
||||||
|
video
|
||||||
|
}
|
||||||
|
MediaItemPayload::Animation { has_spoiler, .. } => {
|
||||||
|
animation_media(item.input_file()?, item_caption, *has_spoiler)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,437 @@
|
|||||||
|
//! Everything around a send: the link-cache write that follows one, the
|
||||||
|
//! keep-alive registry for locally produced media, task settlement, the
|
||||||
|
//! post-send actions (edit prompt / channel forward) and the queue entry
|
||||||
|
//! points.
|
||||||
|
|
||||||
|
use super::{SendError, Task, forward_messages, send_animation, send_media_sequence};
|
||||||
|
use crate::ctx::AppContext;
|
||||||
|
use crate::db::{now_f64, unix_now};
|
||||||
|
use crate::handlers::log_key;
|
||||||
|
use crate::link_cache::{CachedMedia, CachedMediaKind, LinkCache};
|
||||||
|
use crate::media_sender::MediaSender;
|
||||||
|
use crate::queue::{PersistentTaskQueue, QueueError};
|
||||||
|
use crate::state::EditMessage;
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::sync::LazyLock;
|
||||||
|
use teloxide::types::{ChatId, InlineKeyboardButton, InlineKeyboardMarkup, Message, MessageId};
|
||||||
|
|
||||||
|
/// Persists a successful send under the post's cache key. Only runs for a
|
||||||
|
/// fresh (non-resumed) task that carried raw cache data with no file ids yet.
|
||||||
|
pub(super) async fn cache_sent_task(ctx: &AppContext<'_>, task: &Task, media: Vec<CachedMedia>) {
|
||||||
|
let Some(cache_data) = task.cache_data() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
if !cache_data.media.is_empty() || media.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let mut post = cache_data.clone();
|
||||||
|
post.media = media;
|
||||||
|
if let Some(key) = x_media::site::cache_key(&post.url) {
|
||||||
|
ctx.link_cache.put(&key, &post).await;
|
||||||
|
log::debug!("cached send for [key={}]", log_key(&post.url));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Persists a lone animation send under the post's cache key.
|
||||||
|
pub(super) async fn cache_animation_send(ctx: &AppContext<'_>, task: &Task, message: &Message) {
|
||||||
|
if let Some(file_id) = message.animation().map(|a| a.file.id.to_string()) {
|
||||||
|
cache_sent_task(
|
||||||
|
ctx,
|
||||||
|
task,
|
||||||
|
vec![CachedMedia {
|
||||||
|
kind: CachedMediaKind::Animation,
|
||||||
|
file_id,
|
||||||
|
}],
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// How a task ended. The two states differ only in whether a link-cache entry
|
||||||
|
/// may still be holding the (now unusable) media.
|
||||||
|
pub(crate) enum Settled {
|
||||||
|
Sent,
|
||||||
|
Failed,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Every path that ends a task's life — sent, permanently failed, or
|
||||||
|
/// dead-lettered after the last retry — funnels through here, so the cleanup a
|
||||||
|
/// settled task owes cannot be forgotten by a new path: release the keep-alive
|
||||||
|
/// temp media (retryable tasks keep it, they will be resent) and drop the
|
||||||
|
/// link-cache entry that a failed send's stale file ids would keep poisoning.
|
||||||
|
pub(crate) async fn settle_task(ctx: &AppContext<'_>, task: &Task, outcome: Settled) {
|
||||||
|
if matches!(outcome, Settled::Failed) {
|
||||||
|
invalidate_cache(ctx.link_cache, task).await;
|
||||||
|
}
|
||||||
|
release_keep_alive(task);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A cached Telegram file id failed permanently (stale/expired); drop the
|
||||||
|
/// cache entry so the next request re-fetches instead of repeating it.
|
||||||
|
async fn invalidate_cache(cache: &LinkCache, task: &Task) {
|
||||||
|
if task.is_cached_send()
|
||||||
|
&& let Some(url) = task.source_url()
|
||||||
|
&& let Some(key) = x_media::site::cache_key(url)
|
||||||
|
{
|
||||||
|
log::debug!("removing stale link cache entry for [key={}]", log_key(url));
|
||||||
|
cache.remove(&key).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Locally produced media files (ugoira MP4, bsky remux MP4) whose temp dirs
|
||||||
|
/// must stay alive while their task may be retried by the queue. The fetch
|
||||||
|
/// pipeline hands ownership here via
|
||||||
|
/// [`x_media::site::Fetched::take_keep_alive`] before that
|
||||||
|
/// [`x_media::site::Fetched`] is dropped; a queued retry runs after that drop,
|
||||||
|
/// so without this the local file would be gone by the time the retry sends
|
||||||
|
/// it. Entries are removed when the task settles (see [`release_keep_alive`]).
|
||||||
|
pub(crate) static KEEP_ALIVE: LazyLock<parking_lot::Mutex<Vec<tempfile::TempDir>>> =
|
||||||
|
LazyLock::new(|| parking_lot::Mutex::new(Vec::new()));
|
||||||
|
|
||||||
|
/// Drops the keep-alive temp dirs holding media referenced by `task` (matched
|
||||||
|
/// by path prefix). Called once a task settles — sent or permanently failed —
|
||||||
|
/// so retry-only temp files do not leak; retryable tasks keep them alive.
|
||||||
|
pub(crate) fn release_keep_alive(task: &Task) {
|
||||||
|
let paths = task.local_media_paths();
|
||||||
|
if paths.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let mut alive = KEEP_ALIVE.lock();
|
||||||
|
alive.retain(|dir| {
|
||||||
|
let dir_path = dir.path();
|
||||||
|
!paths.iter().any(|p| p.starts_with(dir_path))
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The edit-before-forward prompt's text. It names both controls and the TTL,
|
||||||
|
/// because the buttons alone left users waiting for a forward that never came
|
||||||
|
/// (nothing is forwarded until Confirm).
|
||||||
|
pub(super) fn edit_prompt_text(ttl: std::time::Duration) -> String {
|
||||||
|
format!(
|
||||||
|
"Reply to edit the caption, or tap a template, then ↩️ Confirm to forward. \
|
||||||
|
Expires in {}. Nothing is forwarded until you confirm.",
|
||||||
|
coarsest_unit(ttl)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Text the prompt is rewritten to once its record expires. The sweep edits
|
||||||
|
/// the prompt in place (see `main`): announcing the expiry with a new message
|
||||||
|
/// would wake the chat up to a full TTL later about a prompt nobody is
|
||||||
|
/// waiting on.
|
||||||
|
pub(crate) const EDIT_PROMPT_EXPIRED_TEXT: &str = "⌛ Expired — nothing was forwarded.";
|
||||||
|
|
||||||
|
/// `24h` / `90m` / `45s`: the coarsest whole unit, so the prompt stays short.
|
||||||
|
fn coarsest_unit(ttl: std::time::Duration) -> String {
|
||||||
|
let secs = ttl.as_secs();
|
||||||
|
if secs >= 3600 {
|
||||||
|
format!("{}h", secs / 3600)
|
||||||
|
} else if secs >= 60 {
|
||||||
|
format!("{}m", secs / 60)
|
||||||
|
} else {
|
||||||
|
format!("{secs}s")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Templates per keyboard row. Telegram rejects a keyboard with more than 100
|
||||||
|
/// buttons *outright*, which would silently drop the whole prompt, so the
|
||||||
|
/// names are folded and capped rather than listed one per row.
|
||||||
|
pub(super) const TEMPLATE_BUTTONS_PER_ROW: usize = 3;
|
||||||
|
/// Hard cap on template buttons; the prompt text names the ones not shown.
|
||||||
|
pub(super) const MAX_TEMPLATE_BUTTONS: usize = 60;
|
||||||
|
|
||||||
|
/// Template buttons ([`TEMPLATE_BUTTONS_PER_ROW`] per row, at most
|
||||||
|
/// [`MAX_TEMPLATE_BUTTONS`]), then the confirm/skip pair. Sorted by name: the
|
||||||
|
/// templates live in a `HashMap`, so an unsorted walk would reshuffle the
|
||||||
|
/// buttons between prompts.
|
||||||
|
pub(super) fn build_edit_markup(templates: &HashMap<String, String>) -> InlineKeyboardMarkup {
|
||||||
|
let mut names: Vec<&String> = templates.keys().collect();
|
||||||
|
names.sort();
|
||||||
|
let shown = names.len().min(MAX_TEMPLATE_BUTTONS);
|
||||||
|
let mut rows = Vec::with_capacity(shown / TEMPLATE_BUTTONS_PER_ROW + 2);
|
||||||
|
for chunk in names[..shown].chunks(TEMPLATE_BUTTONS_PER_ROW) {
|
||||||
|
rows.push(
|
||||||
|
chunk
|
||||||
|
.iter()
|
||||||
|
.map(|name| {
|
||||||
|
InlineKeyboardButton::callback(name.as_str(), format!("template|{name}"))
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// Skip exists because the prompt holds the forward hostage until Confirm:
|
||||||
|
// without it the only escape was deleting the message and waiting out the
|
||||||
|
// TTL for a forward that then never happens.
|
||||||
|
rows.push(vec![
|
||||||
|
InlineKeyboardButton::callback("↩️ Confirm", "forward"),
|
||||||
|
InlineKeyboardButton::callback("🛑 Skip", "skip"),
|
||||||
|
]);
|
||||||
|
InlineKeyboardMarkup::new(rows)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// How many templates the markup could not fit, for the prompt text.
|
||||||
|
pub(super) fn hidden_template_count(templates: &HashMap<String, String>) -> usize {
|
||||||
|
templates.len().saturating_sub(MAX_TEMPLATE_BUTTONS)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Notifies a chat about a dead-lettered task (skips when `notify_chat_id` is
|
||||||
|
/// absent).
|
||||||
|
pub(super) async fn notify_failure(
|
||||||
|
sender: &dyn MediaSender,
|
||||||
|
chat_id: Option<i64>,
|
||||||
|
message_id: Option<i64>,
|
||||||
|
message: &str,
|
||||||
|
) {
|
||||||
|
let Some(chat_id) = chat_id else { return };
|
||||||
|
let reply_to = message_id.map(|id| MessageId(id as i32));
|
||||||
|
if let Err(e) = sender
|
||||||
|
.send_message(ChatId(chat_id), message.to_string(), reply_to, None)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
log::error!("failed to notify about failed task: {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// After a successful send: either open the edit-before-forward prompt or
|
||||||
|
/// forward to the configured channel (with retry/queue handling).
|
||||||
|
pub(crate) async fn post_send_actions(ctx: &AppContext<'_>, task: &Task, message_ids: Vec<i64>) {
|
||||||
|
let (
|
||||||
|
chat_id,
|
||||||
|
reply_to,
|
||||||
|
source_url,
|
||||||
|
edit_before_forward,
|
||||||
|
forward_channel_id,
|
||||||
|
notify_chat_id,
|
||||||
|
notify_message_id,
|
||||||
|
) = match task {
|
||||||
|
Task::SendMediaSequence {
|
||||||
|
chat_id,
|
||||||
|
reply_to_message_id,
|
||||||
|
source_url,
|
||||||
|
edit_before_forward,
|
||||||
|
forward_channel_id,
|
||||||
|
notify_chat_id,
|
||||||
|
notify_message_id,
|
||||||
|
..
|
||||||
|
}
|
||||||
|
| Task::SendAnimation {
|
||||||
|
chat_id,
|
||||||
|
reply_to_message_id,
|
||||||
|
source_url,
|
||||||
|
edit_before_forward,
|
||||||
|
forward_channel_id,
|
||||||
|
notify_chat_id,
|
||||||
|
notify_message_id,
|
||||||
|
..
|
||||||
|
} => (
|
||||||
|
*chat_id,
|
||||||
|
*reply_to_message_id,
|
||||||
|
source_url.clone(),
|
||||||
|
*edit_before_forward,
|
||||||
|
*forward_channel_id,
|
||||||
|
*notify_chat_id,
|
||||||
|
*notify_message_id,
|
||||||
|
),
|
||||||
|
Task::ForwardMessages { .. } => return,
|
||||||
|
};
|
||||||
|
|
||||||
|
if edit_before_forward {
|
||||||
|
let templates = ctx.chat_store.get(chat_id).await.template;
|
||||||
|
let keyboard = build_edit_markup(&templates);
|
||||||
|
let mut text = edit_prompt_text(ctx.config.edit_message_ttl);
|
||||||
|
let hidden = hidden_template_count(&templates);
|
||||||
|
if hidden > 0 {
|
||||||
|
// The keyboard is capped; say so instead of silently hiding them.
|
||||||
|
text.push_str(&format!(
|
||||||
|
"\n({hidden} more templates not shown — /remove_template to prune.)"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let prompt = ctx
|
||||||
|
.sender
|
||||||
|
.send_message(
|
||||||
|
ChatId(chat_id),
|
||||||
|
text,
|
||||||
|
Some(MessageId(reply_to as i32)),
|
||||||
|
Some(keyboard),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
match prompt {
|
||||||
|
Ok(prompt_id) => {
|
||||||
|
log::info!(
|
||||||
|
"edit-before-forward prompt {prompt_id} opened for {} message(s)",
|
||||||
|
message_ids.len()
|
||||||
|
);
|
||||||
|
let source_url = source_url.clone();
|
||||||
|
ctx.chat_store
|
||||||
|
.update(chat_id, move |data| {
|
||||||
|
data.edit_message.insert(
|
||||||
|
prompt_id,
|
||||||
|
EditMessage {
|
||||||
|
url: source_url,
|
||||||
|
chat_id,
|
||||||
|
forward_message_ids: message_ids,
|
||||||
|
template: String::new(),
|
||||||
|
created_at: unix_now(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
Err(e) => log::error!("failed to send edit prompt: {e}"),
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(channel_id) = forward_channel_id {
|
||||||
|
log::info!(
|
||||||
|
"forwarding {} message(s) to channel {channel_id}",
|
||||||
|
message_ids.len()
|
||||||
|
);
|
||||||
|
let forward_task = Task::ForwardMessages {
|
||||||
|
from_chat_id: chat_id,
|
||||||
|
to_chat_id: channel_id,
|
||||||
|
message_ids,
|
||||||
|
notify_chat_id,
|
||||||
|
notify_message_id,
|
||||||
|
};
|
||||||
|
match forward_messages(ctx, &forward_task).await {
|
||||||
|
Ok(()) => {}
|
||||||
|
Err(SendError::Retryable {
|
||||||
|
delay_seconds,
|
||||||
|
task,
|
||||||
|
}) => {
|
||||||
|
enqueue_retry(ctx.task_queue, *task, delay_seconds).await;
|
||||||
|
}
|
||||||
|
Err(SendError::Permanent { message, .. }) => {
|
||||||
|
notify_failure(
|
||||||
|
ctx.sender,
|
||||||
|
notify_chat_id,
|
||||||
|
notify_message_id,
|
||||||
|
&failure_text(None, &message),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Enqueues a task for a later attempt (retry / forward resume). When the
|
||||||
|
/// enqueue itself fails the task can never be sent again, so its keep-alive
|
||||||
|
/// temp media is released instead of leaking until process exit.
|
||||||
|
pub(crate) async fn enqueue_retry(queue: &PersistentTaskQueue, task: Task, delay_seconds: f64) {
|
||||||
|
let payload = serde_json::to_value(&task).expect("task serializes");
|
||||||
|
let run_after = now_f64() + delay_seconds;
|
||||||
|
if let Err(e) = queue.enqueue(payload, run_after).await {
|
||||||
|
log::error!("failed to enqueue retry: {e}");
|
||||||
|
release_keep_alive(&task);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Queue entry point: parses the stored task and dispatches.
|
||||||
|
pub(crate) async fn handle_task(
|
||||||
|
ctx: &AppContext<'_>,
|
||||||
|
payload: serde_json::Value,
|
||||||
|
) -> Result<(), QueueError> {
|
||||||
|
let task: Task = match serde_json::from_value(payload.clone()) {
|
||||||
|
Ok(task) => task,
|
||||||
|
Err(e) => {
|
||||||
|
return Err(QueueError::Permanent {
|
||||||
|
message: format!("invalid task payload: {e}"),
|
||||||
|
payload,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
match task {
|
||||||
|
Task::SendMediaSequence { .. } | Task::SendAnimation { .. } => {
|
||||||
|
let message_ids = match send_media_or_animation(ctx, &task).await {
|
||||||
|
Ok(ids) => ids,
|
||||||
|
Err(SendError::Retryable {
|
||||||
|
delay_seconds,
|
||||||
|
task,
|
||||||
|
}) => {
|
||||||
|
return Err(QueueError::Retryable {
|
||||||
|
delay_seconds,
|
||||||
|
payload: serde_json::to_value(task).expect("task serializes"),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Err(SendError::Permanent { message, task }) => {
|
||||||
|
settle_task(ctx, &task, Settled::Failed).await;
|
||||||
|
return Err(QueueError::Permanent {
|
||||||
|
message,
|
||||||
|
payload: serde_json::to_value(task).expect("task serializes"),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
// A task only reaches the queue after a failed send, so this
|
||||||
|
// successful run is the first time post_send_actions can fire —
|
||||||
|
// the fresh attempt failed before it ever got here. Run it
|
||||||
|
// unconditionally: `post_send_actions` executes once, after the
|
||||||
|
// whole sequence (every batch) completed, so the channel forward
|
||||||
|
// and the edit-before-forward prompt must not be lost just
|
||||||
|
// because the send needed a retry.
|
||||||
|
post_send_actions(ctx, &task, message_ids).await;
|
||||||
|
settle_task(ctx, &task, Settled::Sent).await;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
Task::ForwardMessages { .. } => match forward_messages(ctx, &task).await {
|
||||||
|
Ok(()) => Ok(()),
|
||||||
|
Err(SendError::Retryable {
|
||||||
|
delay_seconds,
|
||||||
|
task,
|
||||||
|
}) => Err(QueueError::Retryable {
|
||||||
|
delay_seconds,
|
||||||
|
payload: serde_json::to_value(task).expect("task serializes"),
|
||||||
|
}),
|
||||||
|
Err(SendError::Permanent { message, task }) => {
|
||||||
|
settle_task(ctx, &task, Settled::Failed).await;
|
||||||
|
Err(QueueError::Permanent {
|
||||||
|
message,
|
||||||
|
payload: serde_json::to_value(task).expect("task serializes"),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn send_media_or_animation(ctx: &AppContext<'_>, task: &Task) -> Result<Vec<i64>, SendError> {
|
||||||
|
match task {
|
||||||
|
Task::SendMediaSequence { .. } => send_media_sequence(ctx, task).await,
|
||||||
|
Task::SendAnimation { .. } => send_animation(ctx, task).await,
|
||||||
|
Task::ForwardMessages { .. } => unreachable!(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// User-facing text for a task that will never run again: which link died and
|
||||||
|
/// why. The raw error alone left the user guessing which post it was about.
|
||||||
|
pub(super) fn failure_text(task: Option<&Task>, message: &str) -> String {
|
||||||
|
match task.and_then(|task| task.source_url()).map(log_key) {
|
||||||
|
Some(key) => format!("Send failed permanently for {key}: {message}"),
|
||||||
|
// `ForwardMessages` carries no source URL: that failure is about the
|
||||||
|
// channel copy, not about a post.
|
||||||
|
None => format!("Forward failed permanently: {message}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Dead-letter callback wired to the queue in main: settles the task and
|
||||||
|
/// notifies its chat.
|
||||||
|
pub(crate) async fn dead_letter_notify(
|
||||||
|
ctx: &AppContext<'_>,
|
||||||
|
payload: serde_json::Value,
|
||||||
|
message: String,
|
||||||
|
) {
|
||||||
|
// A dead-lettered task never runs again, and the queue dead-letters retry
|
||||||
|
// exhaustion itself (the handler is not called again), so this is the only
|
||||||
|
// place that sees the final payload.
|
||||||
|
let task = serde_json::from_value::<Task>(payload.clone()).ok();
|
||||||
|
if let Some(task) = &task {
|
||||||
|
settle_task(ctx, task, Settled::Failed).await;
|
||||||
|
}
|
||||||
|
let notify_chat_id = payload.get("notify_chat_id").and_then(|v| v.as_i64());
|
||||||
|
let notify_message_id = payload.get("notify_message_id").and_then(|v| v.as_i64());
|
||||||
|
notify_failure(
|
||||||
|
ctx.sender,
|
||||||
|
notify_chat_id,
|
||||||
|
notify_message_id,
|
||||||
|
&failure_text(task.as_ref(), &message),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
@@ -0,0 +1,345 @@
|
|||||||
|
//! Download-and-reupload fallback: when Telegram cannot fetch a media URL
|
||||||
|
//! itself (hotlink protection), the bot downloads the file, shrinks photos
|
||||||
|
//! that exceed Telegram's limits and uploads the batch via multipart.
|
||||||
|
|
||||||
|
use super::input_media::{animation_media, input_file_for, item_url, photo_media, video_media};
|
||||||
|
use super::{MediaItemPayload, SendError, Task, classify_to_send_error, retry_delay_seconds};
|
||||||
|
use crate::media_sender::MediaSender;
|
||||||
|
use crate::photo::{self, MAX_UPLOAD_BYTES, PhotoPrep};
|
||||||
|
use teloxide::prelude::*;
|
||||||
|
use teloxide::types::{ChatId, InputFile, InputMedia, MessageId};
|
||||||
|
use tempfile::NamedTempFile;
|
||||||
|
use x_media::site::FetchError;
|
||||||
|
|
||||||
|
/// Infers a file extension from magic bytes so Telegram detects the mime type
|
||||||
|
/// on multipart uploads.
|
||||||
|
pub(super) fn sniff_ext(bytes: &[u8]) -> &'static str {
|
||||||
|
if bytes.starts_with(&[0xFF, 0xD8]) {
|
||||||
|
"jpg"
|
||||||
|
} else if bytes.starts_with(b"\x89PNG") {
|
||||||
|
"png"
|
||||||
|
} else if bytes.starts_with(b"RIFF") && bytes.len() >= 12 && &bytes[8..12] == b"WEBP" {
|
||||||
|
"webp"
|
||||||
|
} else if bytes.starts_with(b"GIF8") {
|
||||||
|
"gif"
|
||||||
|
} else if bytes.len() >= 12 && &bytes[4..8] == b"ftyp" {
|
||||||
|
"mp4"
|
||||||
|
} else {
|
||||||
|
"bin"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) enum FallbackError {
|
||||||
|
Retryable {
|
||||||
|
delay_seconds: f64,
|
||||||
|
},
|
||||||
|
Permanent {
|
||||||
|
message: String,
|
||||||
|
},
|
||||||
|
/// The downloaded file exceeds the upload cap; the caller falls back to
|
||||||
|
/// the item's smaller URL.
|
||||||
|
MediaTooLarge,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Brings a downloaded photo within Telegram's limits via the pure-Rust
|
||||||
|
/// chain in [`crate::photo`] (no ffmpeg): dimension cap / upload cap
|
||||||
|
/// exceeded photos are decoded, downscaled with Lanczos3, PNG bit depth
|
||||||
|
/// reduced (>24-bit → 24-bit RGB, ≤24-bit untouched) and transcoded to JPEG
|
||||||
|
/// only if still too big. Anything that cannot be fixed falls back to the
|
||||||
|
/// item's smaller URL.
|
||||||
|
///
|
||||||
|
/// Downloads one media item to a temp file (deleted on drop), returning the
|
||||||
|
/// file plus the downloaded bytes (photos keep the bytes for
|
||||||
|
/// [`photo::prepare_photo`] — re-reading the file would double the I/O).
|
||||||
|
/// Network errors are retryable; size over the upload cap and other download
|
||||||
|
/// errors are not.
|
||||||
|
async fn download_to_temp(
|
||||||
|
item: &MediaItemPayload,
|
||||||
|
) -> Result<(NamedTempFile, bytes::Bytes), FallbackError> {
|
||||||
|
let media_url = match item {
|
||||||
|
MediaItemPayload::Photo { media, .. }
|
||||||
|
| MediaItemPayload::Video { media, .. }
|
||||||
|
| MediaItemPayload::Animation { media, .. } => media,
|
||||||
|
};
|
||||||
|
// Photos are downloaded even over the upload cap so `prepare_photo` can
|
||||||
|
// downscale / transcode them (cap = decode budget); videos/animations
|
||||||
|
// abort as soon as the upload cap is crossed mid-stream.
|
||||||
|
let limit = if matches!(item, MediaItemPayload::Photo { .. }) {
|
||||||
|
photo::MAX_DECODE_BYTES
|
||||||
|
} else {
|
||||||
|
MAX_UPLOAD_BYTES + 1
|
||||||
|
};
|
||||||
|
let bytes = match x_media::site::download_media_limited(media_url, limit).await {
|
||||||
|
Ok(bytes) => bytes,
|
||||||
|
Err(FetchError::Http(_)) => {
|
||||||
|
return Err(FallbackError::Retryable {
|
||||||
|
delay_seconds: retry_delay_seconds(0),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Err(FetchError::TooLarge) => {
|
||||||
|
return Err(FallbackError::MediaTooLarge);
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
return Err(FallbackError::Permanent {
|
||||||
|
message: format!("download failed: {e}"),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let ext = sniff_ext(&bytes);
|
||||||
|
let mut file = tempfile::Builder::new()
|
||||||
|
.suffix(&format!(".{ext}"))
|
||||||
|
.tempfile()
|
||||||
|
.map_err(|e| FallbackError::Permanent {
|
||||||
|
message: format!("temp file failed: {e}"),
|
||||||
|
})?;
|
||||||
|
use std::io::Write;
|
||||||
|
file.as_file_mut()
|
||||||
|
.write_all(&bytes)
|
||||||
|
.map_err(|e| FallbackError::Permanent {
|
||||||
|
message: format!("temp file write failed: {e}"),
|
||||||
|
})?;
|
||||||
|
Ok((file, bytes))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Builds the media group item from an uploaded file.
|
||||||
|
fn media_from_file(
|
||||||
|
item: &MediaItemPayload,
|
||||||
|
path: std::path::PathBuf,
|
||||||
|
caption: Option<&str>,
|
||||||
|
thumbnail: Option<&str>,
|
||||||
|
) -> Result<InputMedia, String> {
|
||||||
|
let mut media = match item {
|
||||||
|
MediaItemPayload::Photo { has_spoiler, .. } => {
|
||||||
|
photo_media(InputFile::file(path), caption, *has_spoiler)
|
||||||
|
}
|
||||||
|
MediaItemPayload::Video { has_spoiler, .. } => {
|
||||||
|
video_media(InputFile::file(path), caption, *has_spoiler)
|
||||||
|
}
|
||||||
|
MediaItemPayload::Animation { has_spoiler, .. } => {
|
||||||
|
animation_media(InputFile::file(path), caption, *has_spoiler)
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if let (Some(thumb), InputMedia::Video(v)) = (thumbnail, &mut media) {
|
||||||
|
*v = v.clone().thumbnail(input_file_for(thumb)?);
|
||||||
|
}
|
||||||
|
Ok(media)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Builds the media group item from a (smaller) URL.
|
||||||
|
fn media_from_url(
|
||||||
|
item: &MediaItemPayload,
|
||||||
|
url: &str,
|
||||||
|
caption: Option<&str>,
|
||||||
|
thumbnail: Option<&str>,
|
||||||
|
) -> Result<InputMedia, String> {
|
||||||
|
let mut media = match item {
|
||||||
|
MediaItemPayload::Photo { has_spoiler, .. } => {
|
||||||
|
photo_media(input_file_for(url)?, caption, *has_spoiler)
|
||||||
|
}
|
||||||
|
MediaItemPayload::Video { has_spoiler, .. } => {
|
||||||
|
video_media(input_file_for(url)?, caption, *has_spoiler)
|
||||||
|
}
|
||||||
|
MediaItemPayload::Animation { has_spoiler, .. } => {
|
||||||
|
animation_media(input_file_for(url)?, caption, *has_spoiler)
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if let (Some(thumb), InputMedia::Video(v)) = (thumbnail, &mut media) {
|
||||||
|
*v = v.clone().thumbnail(input_file_for(thumb)?);
|
||||||
|
}
|
||||||
|
Ok(media)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One item prepared for the upload fallback: the ready-to-send media plus
|
||||||
|
/// the temp file that must stay on disk until the group request completes.
|
||||||
|
pub(super) struct PreparedItem {
|
||||||
|
/// Original position in the batch (concurrent prep completes out of order).
|
||||||
|
pub(super) index: usize,
|
||||||
|
pub(super) media: InputMedia,
|
||||||
|
pub(super) keep_alive: Option<NamedTempFile>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Downloads / processes one media item for the upload fallback (see
|
||||||
|
/// [`send_batch_via_upload`]). Local files are uploaded directly; oversized
|
||||||
|
/// items fall back to their smaller URL; photos are downscaled/transcoded.
|
||||||
|
pub(super) async fn prepare_upload_item(
|
||||||
|
item: MediaItemPayload,
|
||||||
|
index: usize,
|
||||||
|
caption: Option<&str>,
|
||||||
|
) -> Result<PreparedItem, FallbackError> {
|
||||||
|
// Locally produced files (ugoira / bsky remux MP4): nothing to download
|
||||||
|
// or shrink — upload the file directly. The send is a multipart upload,
|
||||||
|
// so the only remaining failure is an upload-cap error, which is
|
||||||
|
// permanent (a video cannot be re-encoded here).
|
||||||
|
let media_url = item_url(&item);
|
||||||
|
if !media_url.starts_with("http://") && !media_url.starts_with("https://") {
|
||||||
|
let media = media_from_file(
|
||||||
|
&item,
|
||||||
|
std::path::PathBuf::from(media_url),
|
||||||
|
caption,
|
||||||
|
item.thumbnail_url(),
|
||||||
|
)
|
||||||
|
.map_err(|message| FallbackError::Permanent { message })?;
|
||||||
|
return Ok(PreparedItem {
|
||||||
|
index,
|
||||||
|
media,
|
||||||
|
keep_alive: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
// Size check before downloading/uploading: over the cap, use the
|
||||||
|
// smaller URL instead of the file. Photos are exempt — they are
|
||||||
|
// downloaded and processed (downscale / PNG→JPEG) before uploading.
|
||||||
|
let too_large = match x_media::site::media_size(media_url).await {
|
||||||
|
Ok(Some(size)) => size > MAX_UPLOAD_BYTES,
|
||||||
|
_ => false,
|
||||||
|
};
|
||||||
|
let too_large = too_large && !matches!(item, MediaItemPayload::Photo { .. });
|
||||||
|
if too_large {
|
||||||
|
let url = item
|
||||||
|
.fallback_url()
|
||||||
|
.ok_or_else(|| FallbackError::Permanent {
|
||||||
|
message: "media too large".into(),
|
||||||
|
})?;
|
||||||
|
let media = media_from_url(&item, url, caption, item.thumbnail_url())
|
||||||
|
.map_err(|message| FallbackError::Permanent { message })?;
|
||||||
|
return Ok(PreparedItem {
|
||||||
|
index,
|
||||||
|
media,
|
||||||
|
keep_alive: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
match download_to_temp(&item).await {
|
||||||
|
Ok((file, bytes)) => {
|
||||||
|
if matches!(item, MediaItemPayload::Photo { .. }) {
|
||||||
|
// Telegram rejects photos wider+taller than 10000 px combined
|
||||||
|
// (PHOTO_INVALID_DIMENSIONS): downscale the downloaded file
|
||||||
|
// before uploading; photos that cannot be brought within the
|
||||||
|
// limits degrade to the smaller URL. CPU-heavy work runs off
|
||||||
|
// the async executor thread.
|
||||||
|
let prep = tokio::task::spawn_blocking(move || photo::prepare_photo(file, &bytes))
|
||||||
|
.await
|
||||||
|
.map_err(|e| FallbackError::Permanent {
|
||||||
|
message: format!("photo worker panicked: {e}"),
|
||||||
|
})?
|
||||||
|
.map_err(|message| FallbackError::Permanent { message })?;
|
||||||
|
match prep {
|
||||||
|
PhotoPrep::Upload(upload) => {
|
||||||
|
let path = upload.path().to_path_buf();
|
||||||
|
let media = media_from_file(&item, path, caption, item.thumbnail_url())
|
||||||
|
.map_err(|message| FallbackError::Permanent { message })?;
|
||||||
|
Ok(PreparedItem {
|
||||||
|
index,
|
||||||
|
media,
|
||||||
|
keep_alive: Some(upload),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
PhotoPrep::UseFallback => {
|
||||||
|
let url = item.fallback_url().ok_or_else(|| FallbackError::Permanent {
|
||||||
|
message: "photo dimensions exceed Telegram limits and no smaller variant is available"
|
||||||
|
.into(),
|
||||||
|
})?;
|
||||||
|
let media = media_from_url(&item, url, caption, item.thumbnail_url())
|
||||||
|
.map_err(|message| FallbackError::Permanent { message })?;
|
||||||
|
Ok(PreparedItem {
|
||||||
|
index,
|
||||||
|
media,
|
||||||
|
keep_alive: None,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
let path = file.path().to_path_buf();
|
||||||
|
let media = media_from_file(&item, path, caption, item.thumbnail_url())
|
||||||
|
.map_err(|message| FallbackError::Permanent { message })?;
|
||||||
|
Ok(PreparedItem {
|
||||||
|
index,
|
||||||
|
media,
|
||||||
|
keep_alive: Some(file),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(FallbackError::MediaTooLarge) => {
|
||||||
|
let url = item
|
||||||
|
.fallback_url()
|
||||||
|
.ok_or_else(|| FallbackError::Permanent {
|
||||||
|
message: "media too large".into(),
|
||||||
|
})?;
|
||||||
|
let media = media_from_url(&item, url, caption, item.thumbnail_url())
|
||||||
|
.map_err(|message| FallbackError::Permanent { message })?;
|
||||||
|
Ok(PreparedItem {
|
||||||
|
index,
|
||||||
|
media,
|
||||||
|
keep_alive: None,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
Err(e) => Err(e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Download-and-reupload fallback for one media batch. Files over the upload
|
||||||
|
/// cap are not downloaded/uploaded; the item falls back to its smaller URL
|
||||||
|
/// (which Telegram fetches itself). Items are prepared concurrently (bounded)
|
||||||
|
/// because the downloads are network-bound; the batch is then uploaded in its
|
||||||
|
/// original order. Returns the fallback-error without the task attached;
|
||||||
|
/// callers wrap it with the updated task state.
|
||||||
|
pub(super) async fn send_batch_via_upload(
|
||||||
|
sender: &dyn MediaSender,
|
||||||
|
chat_id: i64,
|
||||||
|
reply_to: i64,
|
||||||
|
batch: &[MediaItemPayload],
|
||||||
|
caption: Option<&str>,
|
||||||
|
task: Task,
|
||||||
|
) -> Result<Vec<Message>, SendError> {
|
||||||
|
let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(3));
|
||||||
|
let mut set = tokio::task::JoinSet::new();
|
||||||
|
for (i, item) in batch.iter().enumerate() {
|
||||||
|
let item_caption = if i == 0 {
|
||||||
|
caption.map(str::to_string)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
let item = item.clone();
|
||||||
|
let sem = std::sync::Arc::clone(&sem);
|
||||||
|
set.spawn(async move {
|
||||||
|
let _permit = sem.acquire().await.expect("upload semaphore closed");
|
||||||
|
prepare_upload_item(item, i, item_caption.as_deref()).await
|
||||||
|
});
|
||||||
|
}
|
||||||
|
let mut prepared: Vec<Option<InputMedia>> = (0..batch.len()).map(|_| None).collect();
|
||||||
|
let mut keep_alive: Vec<NamedTempFile> = Vec::new();
|
||||||
|
while let Some(joined) = set.join_next().await {
|
||||||
|
let item = match joined {
|
||||||
|
Ok(Ok(item)) => item,
|
||||||
|
// Dropping the JoinSet aborts the remaining prep tasks; their
|
||||||
|
// temp files are cleaned up on drop (short-circuit like before).
|
||||||
|
Ok(Err(e)) => return Err(SendError::from_fallback(e, task.clone())),
|
||||||
|
Err(e) => {
|
||||||
|
return Err(SendError::Permanent {
|
||||||
|
message: format!("upload worker panicked: {e}"),
|
||||||
|
task: Box::new(task),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let PreparedItem {
|
||||||
|
index,
|
||||||
|
media,
|
||||||
|
keep_alive: file_opt,
|
||||||
|
} = item;
|
||||||
|
if let Some(file) = file_opt {
|
||||||
|
keep_alive.push(file);
|
||||||
|
}
|
||||||
|
prepared[index] = Some(media);
|
||||||
|
}
|
||||||
|
let items: Vec<InputMedia> = prepared
|
||||||
|
.into_iter()
|
||||||
|
.map(|m| m.expect("every upload item was prepared"))
|
||||||
|
.collect();
|
||||||
|
// `keep_alive` holds the temp files until the group request completes.
|
||||||
|
let result = sender
|
||||||
|
.send_media_group(ChatId(chat_id), MessageId(reply_to as i32), items)
|
||||||
|
.await;
|
||||||
|
drop(keep_alive);
|
||||||
|
match result {
|
||||||
|
Ok(messages) => Ok(messages),
|
||||||
|
Err(e) => Err(classify_to_send_error(&e, task, "upload failed")),
|
||||||
|
}
|
||||||
|
}
|
||||||
+112
-45
@@ -17,8 +17,8 @@ pub struct ChatData {
|
|||||||
pub edit_message: HashMap<i64, EditMessage>,
|
pub edit_message: HashMap<i64, EditMessage>,
|
||||||
/// name -> HTML template containing "[]"
|
/// name -> HTML template containing "[]"
|
||||||
pub template: HashMap<String, String>,
|
pub template: HashMap<String, String>,
|
||||||
/// site name (twitter/bsky/pixiv) -> user-supplied caption format with
|
/// site name (twitter/bsky/misskey/pixiv/bilibili) -> user-supplied caption format
|
||||||
/// {url} {author} {author_url} {title} {tags} placeholders.
|
/// with {url} {author} {author_url} {title} {content} {tags} placeholders.
|
||||||
pub message_format: HashMap<String, String>,
|
pub message_format: HashMap<String, String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -102,19 +102,22 @@ impl ChatStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The per-chat async lock serializing get→mutate→set cycles.
|
||||||
|
fn lock_for(&self, chat_id: i64) -> Arc<tokio::sync::Mutex<()>> {
|
||||||
|
self.locks
|
||||||
|
.lock()
|
||||||
|
.entry(chat_id)
|
||||||
|
.or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
|
||||||
|
.clone()
|
||||||
|
}
|
||||||
|
|
||||||
/// Serializes a get→mutate→set cycle per chat: concurrent handler tasks
|
/// Serializes a get→mutate→set cycle per chat: concurrent handler tasks
|
||||||
/// (the batch-forward design spawns several per chat) each snapshot the
|
/// (the batch-forward design spawns several per chat) each snapshot the
|
||||||
/// same `ChatData` and last-writer-wins would silently drop mutations,
|
/// same `ChatData` and last-writer-wins would silently drop mutations,
|
||||||
/// e.g. a second `edit_message` record. The per-chat lock makes the
|
/// e.g. a second `edit_message` record. The per-chat lock makes the
|
||||||
/// cycle atomic. Returns the closure's result.
|
/// cycle atomic. Returns the closure's result.
|
||||||
pub async fn update<R>(&self, chat_id: i64, f: impl FnOnce(&mut ChatData) -> R) -> R {
|
pub async fn update<R>(&self, chat_id: i64, f: impl FnOnce(&mut ChatData) -> R) -> R {
|
||||||
let lock = {
|
let lock = self.lock_for(chat_id);
|
||||||
let mut locks = self.locks.lock();
|
|
||||||
locks
|
|
||||||
.entry(chat_id)
|
|
||||||
.or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
|
|
||||||
.clone()
|
|
||||||
};
|
|
||||||
let _guard = lock.lock().await;
|
let _guard = lock.lock().await;
|
||||||
let mut data = self.get(chat_id).await;
|
let mut data = self.get(chat_id).await;
|
||||||
let r = f(&mut data);
|
let r = f(&mut data);
|
||||||
@@ -128,43 +131,45 @@ impl ChatStore {
|
|||||||
pub async fn prune_expired(&self, ttl: Duration) -> Vec<(i64, i64)> {
|
pub async fn prune_expired(&self, ttl: Duration) -> Vec<(i64, i64)> {
|
||||||
let now = unix_now();
|
let now = unix_now();
|
||||||
let ttl_secs = ttl.as_secs() as i64;
|
let ttl_secs = ttl.as_secs() as i64;
|
||||||
let mut removed = Vec::new();
|
// Chats that may have an expired record, from a cache snapshot; the
|
||||||
// Chats with no live edit records: evicted from the cache (and their
|
// pruning itself re-reads and writes under the per-chat lock below
|
||||||
// per-chat lock) so the cache stays bounded to active prompts. The DB
|
// (see the eviction note). Takes no lock of its own, so a chat
|
||||||
// keeps the row; the next get() reloads it.
|
// appearing later is simply picked up by the next sweep.
|
||||||
let mut evicted_chats = Vec::new();
|
let candidates: Vec<i64> = {
|
||||||
let changed: Vec<(i64, ChatData)> = {
|
let cache = self.cache.lock();
|
||||||
let mut cache = self.cache.lock();
|
cache
|
||||||
let mut out = Vec::new();
|
.iter()
|
||||||
for (chat_id, data) in cache.iter_mut() {
|
.filter(|(_, data)| {
|
||||||
let keys: Vec<i64> = data.edit_message.keys().copied().collect();
|
data.edit_message
|
||||||
let mut kept = HashMap::new();
|
.values()
|
||||||
for key in keys {
|
.any(|entry| entry.created_at + ttl_secs <= now)
|
||||||
if let Some(entry) = data.edit_message.get(&key) {
|
})
|
||||||
if entry.created_at + ttl_secs > now {
|
.map(|(chat_id, _)| *chat_id)
|
||||||
kept.insert(key, entry.clone());
|
.collect()
|
||||||
} else {
|
|
||||||
removed.push((*chat_id, key));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if kept.len() != data.edit_message.len() {
|
|
||||||
// Persist the pruned row (removes expired records from
|
|
||||||
// the DB too, not just the cache).
|
|
||||||
data.edit_message = kept;
|
|
||||||
out.push((*chat_id, data.clone()));
|
|
||||||
}
|
|
||||||
if data.edit_message.is_empty() {
|
|
||||||
evicted_chats.push(*chat_id);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Lock order: update() takes the per-chat lock before the cache
|
|
||||||
// lock, so prune must not hold the cache lock while taking locks.
|
|
||||||
drop(cache);
|
|
||||||
out
|
|
||||||
};
|
};
|
||||||
for (chat_id, data) in changed {
|
let mut removed = Vec::new();
|
||||||
self.set(chat_id, &data).await;
|
let mut evicted_chats = Vec::new();
|
||||||
|
for chat_id in candidates {
|
||||||
|
let lock = self.lock_for(chat_id);
|
||||||
|
let _guard = lock.lock().await;
|
||||||
|
let mut data = self.get(chat_id).await;
|
||||||
|
let before = data.edit_message.len();
|
||||||
|
data.edit_message.retain(|key, entry| {
|
||||||
|
if entry.created_at + ttl_secs > now {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
removed.push((chat_id, *key));
|
||||||
|
false
|
||||||
|
});
|
||||||
|
if data.edit_message.len() != before {
|
||||||
|
self.set(chat_id, &data).await;
|
||||||
|
}
|
||||||
|
// Chats with no live edit records: evicted from the cache (and
|
||||||
|
// their per-chat lock) so the cache stays bounded to active
|
||||||
|
// prompts. The DB keeps the row; the next get() reloads it.
|
||||||
|
if data.edit_message.is_empty() {
|
||||||
|
evicted_chats.push(chat_id);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if !evicted_chats.is_empty() {
|
if !evicted_chats.is_empty() {
|
||||||
let mut cache = self.cache.lock();
|
let mut cache = self.cache.lock();
|
||||||
@@ -223,4 +228,66 @@ mod tests {
|
|||||||
"concurrent get→mutate→set must not drop records"
|
"concurrent get→mutate→set must not drop records"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn edit_entry(chat_id: i64, created_at: i64) -> EditMessage {
|
||||||
|
EditMessage {
|
||||||
|
url: "https://x.com/u/status/1".into(),
|
||||||
|
chat_id,
|
||||||
|
forward_message_ids: vec![9],
|
||||||
|
template: String::new(),
|
||||||
|
created_at,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn prune_removes_only_expired_records() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let pool = crate::db::open_store(dir.path().join("p.db").to_str().unwrap()).unwrap();
|
||||||
|
let store = ChatStore::new(pool);
|
||||||
|
let now = unix_now();
|
||||||
|
store
|
||||||
|
.update(7, |data| {
|
||||||
|
data.template.insert("t".into(), "[]".into());
|
||||||
|
data.edit_message.insert(1, edit_entry(7, now - 3600));
|
||||||
|
data.edit_message.insert(2, edit_entry(7, now));
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let removed = store.prune_expired(Duration::from_secs(60)).await;
|
||||||
|
|
||||||
|
assert_eq!(removed, vec![(7, 1)]);
|
||||||
|
let data = store.get(7).await;
|
||||||
|
assert!(data.edit_message.contains_key(&2), "live record pruned");
|
||||||
|
assert_eq!(
|
||||||
|
data.template.get("t").map(String::as_str),
|
||||||
|
Some("[]"),
|
||||||
|
"unrelated state lost by the prune"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn prune_eviction_keeps_the_persisted_state() {
|
||||||
|
// Every record expires → the chat is evicted from the cache; the
|
||||||
|
// pruned state must already be in the DB when that happens.
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let pool = crate::db::open_store(dir.path().join("p.db").to_str().unwrap()).unwrap();
|
||||||
|
let store = ChatStore::new(pool);
|
||||||
|
store
|
||||||
|
.update(8, |data| {
|
||||||
|
data.template.insert("keep".into(), "[]".into());
|
||||||
|
data.edit_message.insert(1, edit_entry(8, 0));
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let removed = store.prune_expired(Duration::from_secs(60)).await;
|
||||||
|
|
||||||
|
assert_eq!(removed, vec![(8, 1)]);
|
||||||
|
let data = store.get(8).await;
|
||||||
|
assert!(data.edit_message.is_empty());
|
||||||
|
assert_eq!(
|
||||||
|
data.template.get("keep").map(String::as_str),
|
||||||
|
Some("[]"),
|
||||||
|
"eviction dropped state the DB never received"
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,6 +10,10 @@
|
|||||||
|
|
||||||
## 1. 现状摩擦清单
|
## 1. 现状摩擦清单
|
||||||
|
|
||||||
|
> ⚠️ 本节记录的是**重构前**的现状:其中的行号、以及 `site/mod.rs` 里的
|
||||||
|
> `fetch_once` 分派函数(当时的实现)都已不存在,仅作历史记录。当前形态见
|
||||||
|
> `site/mod.rs` 的 `SITES` 注册表——新增站点 = 新模块 + 注册一行。
|
||||||
|
|
||||||
以现有三站(twitter / bsky / pixiv)为基线,新增第 4 个站点(代号 `example`)
|
以现有三站(twitter / bsky / pixiv)为基线,新增第 4 个站点(代号 `example`)
|
||||||
今天需要触碰的位置:
|
今天需要触碰的位置:
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user