mirror of
https://github.com/TheFunny/TelegramTwitterMediaBot.git
synced 2026-09-23 23:32:05 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
10a672787a
|
||
|
|
9eb865bb02
|
||
|
|
74ffe66884
|
||
|
|
f8796913e5
|
||
|
|
d60f849864
|
||
|
|
a981256b11
|
@@ -4,6 +4,8 @@ name: CI
|
||||
# job that exercises the real source sites and the token-gated pixiv tests.
|
||||
#
|
||||
# Layering:
|
||||
# changes — decides whether anything but documentation changed; a docs-only
|
||||
# push/PR skips `test` (which then reports as skipped, not missing).
|
||||
# test — fmt + clippy + the full offline unit suite + a release-profile
|
||||
# build + cargo-audit dependency gate. Runs on every push and PR,
|
||||
# including forks (it needs no secrets).
|
||||
@@ -46,7 +48,53 @@ env:
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
jobs:
|
||||
# Docs-only changes skip the heavy job: a README edit does not need a four
|
||||
# minute Rust build (and it cannot break one). A gate job rather than a
|
||||
# workflow-level `paths` filter — that leaves the run without a `test` check
|
||||
# at all, and a required status check then waits for something that will
|
||||
# never be reported, while a *skipped* job reports as neutral.
|
||||
changes:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
outputs:
|
||||
code: ${{ steps.diff.outputs.code }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
fetch-depth: 0 # the diff below needs the pushed range
|
||||
- id: diff
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
zero=0000000000000000000000000000000000000000
|
||||
if [ "${{ github.event_name }}" = "pull_request" ]; then
|
||||
base="origin/${{ github.base_ref }}"
|
||||
git fetch --quiet --no-tags origin "${{ github.base_ref }}"
|
||||
changed="$(git diff --name-only "$base...HEAD")"
|
||||
else
|
||||
before="${{ github.event.before }}"
|
||||
if [ -z "$before" ] || [ "$before" = "$zero" ]; then
|
||||
# New branch or force push: no usable base to compare against,
|
||||
# so the full suite runs. Same for schedule/dispatch, which have
|
||||
# no `before` at all.
|
||||
changed=""
|
||||
else
|
||||
changed="$(git diff --name-only "$before..${{ github.sha }}")"
|
||||
fi
|
||||
fi
|
||||
# Only a change that is *entirely* markdown may skip the job;
|
||||
# anything else — and an empty diff, i.e. a re-run of the same
|
||||
# commit — counts as code.
|
||||
code=true
|
||||
if [ -n "$changed" ] && ! grep -qvE '\.md$' <<<"$changed"; then
|
||||
code=false
|
||||
fi
|
||||
echo "changed: ${changed:-<no diff>}"
|
||||
echo "code=$code" >> "$GITHUB_OUTPUT"
|
||||
|
||||
test:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.code == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
# Generous on purpose: the release-profile build below is cold on the very
|
||||
# first run (thin LTO + codegen-units = 1 across every dependency), and a
|
||||
|
||||
@@ -39,7 +39,9 @@ concurrency:
|
||||
jobs:
|
||||
# A tag push and a branch push to the same commit fire two workflow runs;
|
||||
# build only once. Tag runs always build; master runs build only when the
|
||||
# pushed commit is not already tagged (the tag run covers it).
|
||||
# pushed commit is not already tagged (the tag run covers it). That check
|
||||
# can only see tags that already exist on the remote — see the check step's
|
||||
# re-fetch and the one-push release flow in AGENTS.md.
|
||||
should-build:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
@@ -73,12 +75,38 @@ jobs:
|
||||
- id: check
|
||||
shell: bash
|
||||
run: |
|
||||
if [ "$GITHUB_REF_TYPE" = "branch" ] && git tag --points-at "$GITHUB_SHA" | grep -q .; then
|
||||
zero=0000000000000000000000000000000000000000
|
||||
if [ "$GITHUB_REF_TYPE" = "branch" ]; then
|
||||
# A branch run can start before the release tag for its commit
|
||||
# reaches the remote — pushing master first is the usual way to hit
|
||||
# it — and then `git tag --points-at` legitimately finds nothing
|
||||
# and this run builds the same commit the tag run is building: two
|
||||
# docker builds, one release. (Seen on v1.9.0 and v1.9.1: the
|
||||
# branch run's checkout had every tag *except* the one being
|
||||
# pushed.) Re-fetching here, immediately before the decision,
|
||||
# shrinks the window to "the tag was pushed after this step ran";
|
||||
# pushing the branch and the tag together
|
||||
# (`git push origin master vX.Y.Z`) removes it.
|
||||
git fetch --tags --force --quiet origin
|
||||
if git tag --points-at "$GITHUB_SHA" | grep -q .; then
|
||||
echo "commit already tagged; the tag run builds the image"
|
||||
echo "build=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "build=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
# Nothing the image is made of changed — a documentation or
|
||||
# workflow-only commit — so there is no new image to publish. The
|
||||
# PR trigger's path list plus the crate sources, which the image
|
||||
# compiles into the binary.
|
||||
before="${{ github.event.before }}"
|
||||
if [ -n "$before" ] && [ "$before" != "$zero" ] \
|
||||
&& ! git diff --name-only "$before..$GITHUB_SHA" \
|
||||
| grep -qE '^(Dockerfile|docker-entrypoint\.sh|\.dockerignore|Cargo\.toml|Cargo\.lock|\.github/workflows/docker\.yml|crates/)'; then
|
||||
echo "no build input changed; skipping the image build"
|
||||
echo "build=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
echo "build=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# No `actions/checkout` here on purpose: `docker/build-push-action` defaults
|
||||
# to the Git context (`https://github.com/<owner>/<repo>.git#<ref>`), so
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
Telegram bot (teloxide) that turns post links from X/Twitter, Pixiv, Bluesky, Misskey (misskey.io), and Bilibili dynamics into media messages (images, video, GIF) with the post's title, author, and tags. It supports batch media splitting, retry with persistence, inline queries, forward-channel rebinding with caption templates, and Pixiv ugoira→MP4 transcoding. README is in Chinese; user-facing bot strings are in English. The project is a Rust port of a Python predecessor (see `queue.rs` comments referencing `utils/task_queue.py`).
|
||||
|
||||
Two-crate Cargo workspace (both v1.9.0, edition 2024, resolver 3):
|
||||
Two-crate Cargo workspace (both v1.9.1, edition 2024, resolver 3):
|
||||
|
||||
- **`crates/x-media`** — library that fetches and normalizes media from the four sites. Pure, no Telegram knowledge.
|
||||
- **`crates/xmedia-bot`** — the bot binary: teloxide dispatcher, SQLite-backed chat state, persistent task queue.
|
||||
@@ -97,7 +97,7 @@ Docker: `docker build -t tgxmb .` then `docker run --rm -d --name tgxmb --env-fi
|
||||
- **Rust, stable, edition 2024**, workspace resolver 3. No `rust-version`/MSRV pin, no `rust-toolchain.toml` — recent stable is assumed. No nightly features.
|
||||
- Package manager: **Cargo** (workspace with path dep `x-media` ← `xmedia-bot`). No `[workspace.package]`/shared deps — each crate lists deps independently.
|
||||
- **TLS is rustls end-to-end** (no native-tls/openssl in the tree, no libssl in the Docker runtime image): `teloxide` is declared `default-features = false` with `["webhooks-axum", "macros", "rustls", "ctrlc_handler"]` (the removed `default` also carried `native-tls` and `ctrlc_handler` — the latter must stay); x-media's reqwest is `default-features = false` with `["json", "rustls-tls", "gzip", "http2"]` (webpki-roots baked in, so the image ships no CA bundle; `gzip` because the site APIs answer their JSON compressed — twitter's syndication body is 4469 bytes identity vs 1066 gzipped — and `http2` because every site CDN here negotiates h2). One reqwest 0.12.28 in the lock.
|
||||
- **Versioning**: bump the version in all three places (`crates/x-media/Cargo.toml`, `crates/xmedia-bot/Cargo.toml`, `Cargo.lock`) and **keep `README.md`, `README.en.md` and `AGENTS.md` in sync with the code on every bump**, then commit (`chore: bump version to X.Y.Z`), create an annotated tag `vX.Y.Z`, and push branch + tag (the tag push triggers the Docker Hub build). The tag must equal both crate versions: `.github/workflows/docker.yml` verifies that before building, and `--locked` verifies the lock file.
|
||||
- **Versioning**: bump the version in all three places (`crates/x-media/Cargo.toml`, `crates/xmedia-bot/Cargo.toml`, `Cargo.lock`) and **keep `README.md`, `README.en.md` and `AGENTS.md` in sync with the code on every bump**, then commit (`chore: bump version to X.Y.Z`), create an annotated tag `vX.Y.Z`, and push branch + tag **in one push** (`git push origin master vX.Y.Z`; the tag push triggers the Docker Hub build). Pushing them separately with the branch first makes the master run of `docker.yml` build the same commit as the tag run — its duplicate check can only see the tags that already exist on the remote. The tag must equal both crate versions: `.github/workflows/docker.yml` verifies that before building, and `--locked` verifies the lock file.
|
||||
- Config is **environment-variable driven** (dotenv loads `.env`, which is gitignored; `.env.example` is the tracked template — `cp .env.example .env` — and is also the file `docker compose` substitutes `${VAR}` from, so every variable the compose passes must be documented there). Key vars: `TELOXIDE_TOKEN` (required), `PIXIV_REFRESH_TOKEN`, `TWITTER_AUTH_TOKEN` (optional; x.com `auth_token` cookie — enables the logged-in GraphQL fallback that fetches NSFW tweets syndication withholds), `BILIBILI_COOKIE` (optional; whole bilibili cookie string — bilibili dynamics fetch anonymously and add their own device cookies, this only rescues an egress IP that bilibili has hard-flagged with `-352`/412), `BOT_ADMIN` (comma-separated ids), `EDIT_MESSAGE_TTL_SECONDS` (default 86400), `LINK_CACHE_TTL_SECONDS` (default 604800), `CAPTION_QUOTE_TEXT_CHARS` (default 200; a post whose text — the `title` plus `content` joined, see `site::compose_text` — reaches this length gets that text wrapped in an expandable blockquote inside its caption, the URL and author line staying outside; `0` disables it. Applied at the send boundary in `send::quote_long_caption`, which locates the text as what follows the author link, so a `/set_format` that moves `{title}`/`{content}` elsewhere and pixiv's title-inside-a-link layout opt out; `copy_messages` forwards and queued retries inherit the wrap, while the edit-before-forward rewrite stays unquoted by design), `DATA_DIR` (default `data`, CWD-relative; the SQLite dir, auto-created), `WEBHOOK`/`WEBHOOK_URL`/`WEBHOOK_LISTEN`/`WEBHOOK_PORT`/`WEBHOOK_CERT`/`WEBHOOK_SECRET_TOKEN` (webhook mode requires URL/listen/port, `.expect`ed; `WEBHOOK_CERT` is Telegram-facing self-signed validation only — TLS must be terminated by a reverse proxy), `RUST_LOG`, `TELOXIDE_PROXY`, `LOCAL_USER_ID` (entrypoint only).
|
||||
- SQLite via `rusqlite` with `bundled` feature (no system libsqlite needed). DB file `$DATA_DIR/task_queue.db` (default `data/task_queue.db`, CWD-relative — run from the workspace root, or `/app` in Docker; set `DATA_DIR` to pin state anywhere). Mount `./data` and `./cert` volumes.
|
||||
- `.gitattributes` enforces LF for `*.sh` (CRLF breaks shebangs in containers). `.gitignore`: `.env`, `data/`, `cert/`, `nginx-*` (proxy state), `/target`, `.idea/` (the compose file is tracked; only `.env` carries the deployment's own values).
|
||||
@@ -109,6 +109,6 @@ Docker: `docker build -t tgxmb .` then `docker run --rm -d --name tgxmb --env-fi
|
||||
- No mocking framework anywhere (no mockito/wiremock/mockall). Conventions: pure-function units (regex parsing, serde round-trips, chunking, retry math) tested synchronously; async tests use real dependencies — file-backed SQLite via `tempfile` (`queue.rs::new_queue()` helper), live network fetches. Tests that must go through a **real `Bot`** (its URL/multipart building, the per-chat limiter and the bot-wide budget) talk to a stand-in API instead (`media_sender::test_support::fake_api::FakeApi`, a `tokio` TCP listener that records every call and answers the smallest result each method needs — teloxide keys methods by payload type, so the recorded name is `SendMediaGroup`, not `sendMediaGroup`): a media group, the edit-before-forward prompt through the real callback path, and `handlers::handle_message` (the context-taking body of `message_handler`, split out for exactly this).
|
||||
- Live-network tests exist in `site/twitter/interface.rs` (5), `site/bsky/interface.rs` (1), `site/misskey/interface.rs` (1), `site/bilibili/interface.rs` (5), `site/pixiv/api.rs` (1); `photo.rs` adds one `#[ignore = "heavy: …"]` test. `site/mod.rs` also has a **token-gated but not `#[ignore]`d** pixiv download test (`download_media_pixiv_original_with_referer`): it hits `i.pximg.net` whenever `PIXIV_REFRESH_TOKEN` is set, so a local `cargo test --workspace` is not fully offline and can flake on a pixiv CDN body timeout. `disabled_site_is_reported_not_ignored` (same file) is gated the other way round: it asserts `fetch` answers `FetchError::Disabled { site: "pixiv" }` for a pixiv link and early-returns when `PIXIV_REFRESH_TOKEN` **is** set (the site is then enabled). Test gating convention (enforced by `.github/workflows/ci.yml`): pure unit tests always run; live-network tests carry `#[ignore = "live network: ..."]` (run via `cargo test --workspace -- --ignored live`); token-gated pixiv tests early-return when `PIXIV_REFRESH_TOKEN` is absent **or empty** (an unset GitHub secret arrives as `""` — `is_err()` alone would run them tokenless and fail), and the bilibili live tests early-return when the API answers risk control (`-352`, which bilibili applies per IP by request volume). Run the full offline suite with `cargo test --workspace`.
|
||||
- Fixtures are inline `serde_json::json!` builder fns (`fixture()`, `thread_json()`, `illust_json()`), not files. The shared `CLIENT` sets `pool_max_idle_per_host(0)` under `#[cfg(test)]` to avoid cross-runtime `DispatchGone`.
|
||||
- **CI** — `.github/workflows/ci.yml` (actions pinned to commit SHAs, `--locked` on every cargo invocation, `concurrency` cancels superseded runs, `RUST_BACKTRACE=1`) runs `cargo fmt --check` + `cargo clippy --workspace --all-targets --locked -- -D warnings` + `cargo test --workspace --locked` + a release-profile `cargo build --release --locked` + an `actions-rust-lang/audit` dependency-vulnerability gate (offline, no secrets, on every push/PR) and a `live` job (schedule/manual/tag only, `-p x-media` since every network/secret-gated test lives there, `continue-on-error`) for the `#[ignore]`d live + token tests. `.github/workflows/docker.yml` builds and pushes the image on master/tag and runs a **build-only check on pull requests touching the build inputs** (`Dockerfile`, entrypoint, manifests, `.dockerignore`); a release tag must match both crate versions or the build stops, and `FFMPEG_URL`/`FFMPEG_SHA256` are taken from repository variables when set (a release can pin an exact ffmpeg build). `.github/dependabot.yml` keeps crates, the pinned actions and the Docker base images current.
|
||||
- **CI** — `.github/workflows/ci.yml` (actions pinned to commit SHAs, `--locked` on every cargo invocation, `concurrency` cancels superseded runs, `RUST_BACKTRACE=1`) runs (behind a `changes` gate job, so a push/PR whose entire diff is markdown skips it instead of burning four minutes on nothing) `cargo fmt --check` + `cargo clippy --workspace --all-targets --locked -- -D warnings` + `cargo test --workspace --locked` + a release-profile `cargo build --release --locked` + an `actions-rust-lang/audit` dependency-vulnerability gate (offline, no secrets, on every push/PR) and a `live` job (schedule/manual/tag only, `-p x-media` since every network/secret-gated test lives there, `continue-on-error`) for the `#[ignore]`d live + token tests. `.github/workflows/docker.yml` builds and pushes the image on master/tag and runs a **build-only check on pull requests touching the build inputs**; its `should-build` gate skips a branch push that is already tagged (`git tag --points-at` — the tag run builds it, so push both refs together) or that touched no build input at all, while a tag push always builds (`Dockerfile`, entrypoint, manifests, `.dockerignore`); a release tag must match both crate versions or the build stops, and `FFMPEG_URL`/`FFMPEG_SHA256` are taken from repository variables when set (a release can pin an exact ffmpeg build). `.github/dependabot.yml` keeps crates, the pinned actions and the Docker base images current.
|
||||
- Untested and hard to test without a mock seam: `config.rs`, `handlers/statics.rs`; `db.rs` is covered for the migration chain but not for pool behaviour under contention; `main.rs` is covered where it was split out (`periodic_sweep`, `sweep_temp_dir`) but not for startup/shutdown or its `dptree` branch tree (the handlers themselves are, through the stand-in API); in `x-media`: `media.rs`, `lib.rs`, all `model.rs`. The `commands.rs` *executor* needs a real `Bot` (only its pure report builder is tested). Everything else — `handlers/{mod,callback,inline,urls}.rs`, `send/*`, `ctx.rs`, `state.rs`, `queue.rs`, `link_cache.rs`, `rate_limit.rs` — is driven through `TestStores`/`ctx::test_support` and the scripted `MockSender`.
|
||||
- No coverage tracking.
|
||||
|
||||
Generated
+11
-11
@@ -275,7 +275,7 @@ version = "3.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -553,7 +553,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -750,9 +750,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "0.4.15"
|
||||
version = "0.4.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155"
|
||||
checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16"
|
||||
dependencies = [
|
||||
"atomic-waker",
|
||||
"bytes",
|
||||
@@ -1153,7 +1153,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1644,7 +1644,7 @@ dependencies = [
|
||||
"once_cell",
|
||||
"socket2",
|
||||
"tracing",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1892,7 +1892,7 @@ dependencies = [
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2318,7 +2318,7 @@ dependencies = [
|
||||
"getrandom 0.4.3",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2727,7 +2727,7 @@ version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2879,7 +2879,7 @@ checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc"
|
||||
|
||||
[[package]]
|
||||
name = "x-media"
|
||||
version = "1.9.0"
|
||||
version = "1.9.1"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"dotenv",
|
||||
@@ -2899,7 +2899,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "xmedia-bot"
|
||||
version = "1.9.0"
|
||||
version = "1.9.1"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"dotenv",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "x-media"
|
||||
version = "1.9.0"
|
||||
version = "1.9.1"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "xmedia-bot"
|
||||
version = "1.9.0"
|
||||
version = "1.9.1"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
|
||||
Reference in New Issue
Block a user