diff --git a/.dockerignore b/.dockerignore index eb9beb3..8d601c5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -26,6 +26,10 @@ *.db LICENSE README.md +# Documentation and scratch files: the build only ever reads the manifests, +# `crates/` and the entrypoint script. +docs/ +*.md data/ cert/ nginx-certs/ diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..be47ddb --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,32 @@ +version: 2 + +# Pairs with the `actions-rust-lang/audit` gate in ci.yml: the gate reports +# advisories in Cargo.lock, this is what actually moves the dependencies. +# Patch bumps are batched into one PR; minor/major stay separate so they get +# reviewed and tested individually. +updates: + - package-ecosystem: cargo + directory: / + schedule: + interval: weekly + open-pull-requests-limit: 5 + groups: + cargo-patch: + applies-to: version-updates + patterns: ['*'] + update-types: ['patch'] + + # The workflow actions are pinned to commit SHAs; that pin is what makes + # bumping them a manual chore, so let the bot do it. + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + open-pull-requests-limit: 5 + + # The Dockerfile's base images (rust:1-bookworm, debian:bookworm-slim). + - package-ecosystem: docker + directory: / + schedule: + interval: monthly + open-pull-requests-limit: 3 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47e7426..b71fb1b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,15 +4,19 @@ name: CI # job that exercises the real source sites and the token-gated pixiv tests. # # Layering: -# test — fmt + clippy + the full offline unit suite + cargo-audit -# dependency gate. Runs on every push and PR, including forks -# (it needs no secrets). +# test — fmt + clippy + the full offline unit suite + a release-profile +# build + cargo-audit dependency gate. Runs on every push and PR, +# including forks (it needs no secrets). # live — the #[ignore]d live-network tests plus the pixiv tests that are # gated on PIXIV_REFRESH_TOKEN. Runs on schedule / manual dispatch # / tag pushes only, because pull requests from forks cannot read # repository secrets. continue-on-error keeps a flaky external site # from blocking, while the run still records the outcome. # +# Every action is pinned to a commit SHA (Dependabot keeps the pins current); +# `dtolnay/rust-toolchain` deliberately stays on its channel ref, because the +# ref itself is what selects the toolchain (`@stable` = install stable). +# # Test gating convention (keep in sync with AGENTS.md "Testing & QA"): # - pure unit tests: plain #[test] / #[tokio::test], always run. # - live-network tests: #[ignore = "live network: ..."], only run here. @@ -28,44 +32,75 @@ on: - cron: '0 3 * * 1' workflow_dispatch: +permissions: + contents: read + +# A newer push to the same ref supersedes the older run; without this every +# intermediate commit of a PR branch keeps a runner busy to completion. +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +env: + # Panicking tests print their backtrace; free when nothing fails. + RUST_BACKTRACE: 1 + jobs: test: runs-on: ubuntu-latest + # Generous on purpose: the release-profile build below is cold on the very + # first run (thin LTO + codegen-units = 1 across every dependency), and a + # timeout there would kill the job *before* rust-cache saves its cache — + # leaving every later run cold again. + timeout-minutes: 45 steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: dtolnay/rust-toolchain@stable with: components: clippy, rustfmt - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 + # `--locked` on every cargo invocation: the version bump edits + # Cargo.lock by hand (AGENTS.md), so a stale lock must fail here instead + # of being silently re-resolved — otherwise CI tests a different + # dependency set than the one committed, and than the one the released + # image is built from. - name: Check formatting run: cargo fmt --check - name: Lint (deny warnings) - run: cargo clippy --workspace --all-targets -- -D warnings + run: cargo clippy --workspace --all-targets --locked -- -D warnings - name: Run offline tests - run: cargo test --workspace + run: cargo test --workspace --locked + # The release profile (lto/strip/codegen-units=1, overflow checks off) + # was otherwise only exercised by the Docker build on master/tag. Same + # package the Dockerfile builds; the cache keeps it cheap after the + # first run. + - name: Build release profile + run: cargo build --release --locked -p xmedia-bot # Dependency vulnerability gate: fails the build when a crate in # Cargo.lock has an unfixed security advisory. Unmaintained/unsound # *warnings* (dotenv, proc-macro-error2, anyhow transitive) do not fail # the build by default; the advisory DB is cached across runs. - name: Audit dependencies - uses: actions-rust-lang/audit@v1 + uses: actions-rust-lang/audit@72c09e02f132669d52284a3323acdb503cfc1a24 # v1 live: needs: test if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/v') runs-on: ubuntu-latest + timeout-minutes: 30 continue-on-error: true env: PIXIV_REFRESH_TOKEN: ${{ secrets.PIXIV_REFRESH_TOKEN }} TWITTER_AUTH_TOKEN: ${{ secrets.TWITTER_AUTH_TOKEN }} steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - # Full suite: with the secret present, the pixiv token-gated tests run; - # without it they skip themselves. Live tests stay #[ignore]d here. + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 + # Everything network- or secret-gated lives in x-media, and the bot + # crate's suite (MockSender + tempdir stores, no network) already ran in + # the `test` job — rebuilding it here bought nothing. - name: Run token-gated tests - run: cargo test --workspace + run: cargo test -p x-media --locked # The live-network tests, by the "live" name filter (all #[ignore]d). - name: Run live-network tests - run: cargo test --workspace -- --ignored live + run: cargo test -p x-media --locked -- --ignored live diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2c94cc4..0699964 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,28 +1,75 @@ name: Build Docker Image +# Release builds (master / v* tags) plus a build-only check on pull requests +# that touch anything the image depends on — the Dockerfile's stub-source +# machinery, the ffmpeg download and the entrypoint are exactly the parts that +# would otherwise break only at release time. +# +# Actions are pinned to commit SHAs (Dependabot keeps the pins current). + on: push: tags: - v* branches: - master + pull_request: + paths: + - Dockerfile + - docker-entrypoint.sh + - .dockerignore + - Cargo.toml + - Cargo.lock + - .github/workflows/docker.yml + - 'crates/**/Cargo.toml' env: APP_NAME: telegram-twitter-media-bot DOCKERHUB_REPO: yoursfunny/telegram-twitter-media-bot +permissions: + contents: read + +# Serialize runs per ref. Never cancel in progress: a killed run would drop a +# half-finished image push. +concurrency: + group: docker-${{ github.ref }} + cancel-in-progress: false + jobs: # A tag push and a branch push to the same commit fire two workflow runs; # build only once. Tag runs always build; master runs build only when the # pushed commit is not already tagged (the tag run covers it). should-build: runs-on: ubuntu-latest + timeout-minutes: 10 outputs: build: ${{ steps.check.outputs.build }} steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 + # A release tag is the version claim: the manifests are bumped by hand, + # so `v1.5.1` with `Cargo.toml` still at 1.5.0 would publish an image + # whose tag lies about what is inside it (the binary carries no version). + - name: Verify the tag matches both crate versions + if: startsWith(github.ref, 'refs/tags/v') + shell: bash + run: | + tag="${GITHUB_REF_NAME#v}" + status=0 + for manifest in crates/x-media/Cargo.toml crates/xmedia-bot/Cargo.toml; do + # tr -d '\r': a CRLF checkout (core.autocrlf on Windows) would + # otherwise yield "1.5.0\r" and false-fail every tag. + version="$(sed -n 's/^version = "\(.*\)"/\1/p' "$manifest" | head -1 | tr -d '\r')" + if [ "$version" != "$tag" ]; then + echo "::error file=$manifest::$manifest is at $version but the tag is v$tag" + status=1 + else + echo "$manifest: $version matches v$tag" + fi + done + exit "$status" - id: check shell: bash run: | @@ -37,10 +84,11 @@ jobs: needs: should-build if: needs.should-build.outputs.build == 'true' runs-on: ubuntu-latest + timeout-minutes: 60 steps: - name: Docker meta id: meta - uses: docker/metadata-action@v6 + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6 with: images: ${{ env.DOCKERHUB_REPO }} tags: | @@ -49,15 +97,15 @@ jobs: type=semver,pattern={{major}}.{{minor}} type=semver,pattern={{major}} type=sha - - - name: Set up QEMU - uses: docker/setup-qemu-action@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4 + # Pull requests build the image to prove the Dockerfile still works, but + # must not read registry credentials (fork PRs have none). - name: Login to Docker Hub - uses: docker/login-action@v4 + if: github.event_name != 'pull_request' + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -66,15 +114,26 @@ jobs: # stage's layers so the cargo-deps and ffmpeg layers are restored # instead of re-downloaded/recompiled. The scope must be pinned to a # fixed string: the gha backend defaults to the current git ref, which - # would give every new tag a cold cache on release builds. + # would give every new tag a cold cache on release builds. PR runs only + # read it (cache-to is empty) so they cannot evict the release cache. + # + # FFMPEG_URL/FFMPEG_SHA256 come from repository variables when set, so a + # release can pin an exact ffmpeg build (the Dockerfile default follows + # the project's `/redirect/latest/` URL, which has no sha256 sidecar). + # + # Single-arch (amd64) on purpose: adding arm64 means re-adding + # `docker/setup-qemu-action`, `platforms: linux/amd64,linux/arm64`, and + # parameterizing FFMPEG_URL by $TARGETARCH in the Dockerfile. - name: Build and push - uses: docker/build-push-action@v7 + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7 with: - push: true + push: ${{ github.event_name != 'pull_request' }} build-args: | APP_NAME=${{ env.APP_NAME }} + FFMPEG_URL=${{ vars.FFMPEG_URL || 'https://ffmpeg.martin-riedl.de/redirect/latest/linux/amd64/release/ffmpeg.zip' }} + FFMPEG_SHA256=${{ vars.FFMPEG_SHA256 }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha,scope=tgxmb-build - cache-to: type=gha,mode=max,scope=tgxmb-build + cache-to: ${{ github.event_name != 'pull_request' && 'type=gha,mode=max,scope=tgxmb-build' || '' }} diff --git a/AGENTS.md b/AGENTS.md index e5fb567..efd83f4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -81,7 +81,7 @@ Docker: `docker build -t tgxmb .` then `docker run --rm -d --name tgxmb --env-fi | `Dockerfile` | Multi-stage: cached dep layer via stub sources + `touch *.rs` mtime bump (cargo's freshness is mtime-based and `cargo clean -p` removes 0 files — the touch is what forces the real sources to rebuild while deps stay cached), static ffmpeg from ffmpeg.martin-riedl.de (`FFMPEG_URL` arg, optional `FFMPEG_SHA256` checksum, `unzip -t` integrity check), `debian:bookworm-slim` runtime, entrypoint. Runtime ships **no libssl/libcrypto/CA bundle** — rustls webpki-roots handles all TLS, and the static ffmpeg only processes local files (downloads go through reqwest) | | `docker-entrypoint.sh` | Privilege drop: `useradd` with `LOCAL_USER_ID` (default 9001) + `setpriv` (no gosu on bookworm-slim) | | `docker-compose.yml.example` | Deployment env reference (real `docker-compose.yml` is gitignored). Ships nginx-proxy + acme-companion: webhook mode needs TLS termination in front (teloxide's axum listener is HTTP-only; `WEBHOOK_CERT` only feeds `set_webhook`), bot exposes `VIRTUAL_HOST`/`VIRTUAL_PORT` on the shared `proxy` network, no host port; container names `nginx-proxy`/`acme-companion`/`tgxmb`, start order via `depends_on` (proxy → acme → bot) | -| `.github/workflows/docker.yml` | CI: build+push to Docker Hub on tag `v*`/master; **no test step**; buildx gha cache (`cache-from`/`cache-to`, scope `tgxmb-build`, `mode=max`) so cargo deps + ffmpeg layers are restored across runs | +| `.github/workflows/docker.yml` | CI: build+push to Docker Hub on tag `v*`/master, plus a build-only check on PRs touching the build inputs; **no test step**; verifies a release tag matches both crate versions; buildx gha cache (`cache-from` always, `cache-to` except on PRs, scope `tgxmb-build`, `mode=max`) so cargo deps + ffmpeg layers are restored across runs; `FFMPEG_URL`/`FFMPEG_SHA256` come from repo variables when set | | `README.md` | Feature docs + command table (Chinese) | ## Runtime/Tooling Preferences @@ -89,7 +89,7 @@ Docker: `docker build -t tgxmb .` then `docker run --rm -d --name tgxmb --env-fi - **Rust, stable, edition 2024**, workspace resolver 3. No `rust-version`/MSRV pin, no `rust-toolchain.toml` — recent stable is assumed. No nightly features. - Package manager: **Cargo** (workspace with path dep `x-media` ← `xmedia-bot`). No `[workspace.package]`/shared deps — each crate lists deps independently. - **TLS is rustls end-to-end** (no native-tls/openssl in the tree, no libssl in the Docker runtime image): `teloxide` is declared `default-features = false` with `["webhooks-axum", "macros", "rustls", "ctrlc_handler"]` (the removed `default` also carried `native-tls` and `ctrlc_handler` — the latter must stay); x-media's reqwest is `default-features = false` with `["json", "rustls-tls"]` (webpki-roots baked in, so the image ships no CA bundle). One reqwest 0.12.28 in the lock. -- **Versioning**: bump the version in all three places (`crates/x-media/Cargo.toml`, `crates/xmedia-bot/Cargo.toml`, `Cargo.lock`) and **keep `README.md`, `README.en.md` and `AGENTS.md` in sync with the code on every bump**, then commit (`chore: bump version to X.Y.Z`), create an annotated tag `vX.Y.Z`, and push branch + tag (the tag push triggers the Docker Hub build). +- **Versioning**: bump the version in all three places (`crates/x-media/Cargo.toml`, `crates/xmedia-bot/Cargo.toml`, `Cargo.lock`) and **keep `README.md`, `README.en.md` and `AGENTS.md` in sync with the code on every bump**, then commit (`chore: bump version to X.Y.Z`), create an annotated tag `vX.Y.Z`, and push branch + tag (the tag push triggers the Docker Hub build). The tag must equal both crate versions: `.github/workflows/docker.yml` verifies that before building, and `--locked` verifies the lock file. - Config is **environment-variable driven** (dotenv loads `.env`, gitignored; no `.env.example` exists). Key vars: `TELOXIDE_TOKEN` (required), `PIXIV_REFRESH_TOKEN`, `TWITTER_AUTH_TOKEN` (optional; x.com `auth_token` cookie — enables the logged-in GraphQL fallback that fetches NSFW tweets syndication withholds), `BOT_ADMIN` (comma-separated ids), `EDIT_MESSAGE_TTL_SECONDS` (default 86400), `LINK_CACHE_TTL_SECONDS` (default 604800), `DATA_DIR` (default `data`, CWD-relative; the SQLite dir, auto-created), `WEBHOOK`/`WEBHOOK_URL`/`WEBHOOK_LISTEN`/`WEBHOOK_PORT`/`WEBHOOK_CERT`/`WEBHOOK_SECRET_TOKEN` (webhook mode requires URL/listen/port, `.expect`ed; `WEBHOOK_CERT` is Telegram-facing self-signed validation only — TLS must be terminated by a reverse proxy), `RUST_LOG`, `TELOXIDE_PROXY`, `LOCAL_USER_ID` (entrypoint only). - SQLite via `rusqlite` with `bundled` feature (no system libsqlite needed). DB file `$DATA_DIR/task_queue.db` (default `data/task_queue.db`, CWD-relative — run from the workspace root, or `/app` in Docker; set `DATA_DIR` to pin state anywhere). Mount `./data` and `./cert` volumes. - `.gitattributes` enforces LF for `*.sh` (CRLF breaks shebangs in containers). `.gitignore`: `.env`, `data/`, `cert/`, `docker-compose.yml`, `/target`, `.idea/`. @@ -101,6 +101,6 @@ Docker: `docker build -t tgxmb .` then `docker run --rm -d --name tgxmb --env-fi - No mocking framework anywhere (no mockito/wiremock/mockall). Conventions: pure-function units (regex parsing, serde round-trips, chunking, retry math) tested synchronously; async tests use real dependencies — file-backed SQLite via `tempfile` (`queue.rs::new_queue()` helper), live network fetches. - Live-network tests exist in `site/twitter/interface.rs` (5), `site/bsky/interface.rs` (2), `site/misskey/interface.rs` (1), `site/pixiv/api.rs` (1); `photo.rs` adds one `#[ignore = "heavy: …"]` test. `site/mod.rs` also has a **token-gated but not `#[ignore]`d** pixiv download test (`download_media_pixiv_original_with_referer`): it hits `i.pximg.net` whenever `PIXIV_REFRESH_TOKEN` is set, so a local `cargo test --workspace` is not fully offline and can flake on a pixiv CDN body timeout. Test gating convention (enforced by `.github/workflows/ci.yml`): pure unit tests always run; live-network tests carry `#[ignore = "live network: ..."]` (run via `cargo test --workspace -- --ignored live`); token-gated pixiv tests early-return when `PIXIV_REFRESH_TOKEN` is absent **or empty** (an unset GitHub secret arrives as `""` — `is_err()` alone would run them tokenless and fail). Run the full offline suite with `cargo test --workspace`. - Fixtures are inline `serde_json::json!` builder fns (`fixture()`, `thread_json()`, `illust_json()`), not files. The shared `CLIENT` sets `pool_max_idle_per_host(0)` under `#[cfg(test)]` to avoid cross-runtime `DispatchGone`. -- **CI** — `.github/workflows/ci.yml` runs `cargo fmt --check` + `cargo clippy --workspace --all-targets -- -D warnings` + `cargo test --workspace` + a `actions-rust-lang/audit` dependency-vulnerability gate (offline, no secrets, on every push/PR) and a `live` job (schedule/manual/tag only, `PIXIV_REFRESH_TOKEN`/`TWITTER_AUTH_TOKEN` from secrets, `continue-on-error`) for the `#[ignore]`d live + token tests. `.github/workflows/docker.yml` builds/pushes the image only. +- **CI** — `.github/workflows/ci.yml` (actions pinned to commit SHAs, `--locked` on every cargo invocation, `concurrency` cancels superseded runs, `RUST_BACKTRACE=1`) runs `cargo fmt --check` + `cargo clippy --workspace --all-targets --locked -- -D warnings` + `cargo test --workspace --locked` + a release-profile `cargo build --release --locked` + an `actions-rust-lang/audit` dependency-vulnerability gate (offline, no secrets, on every push/PR) and a `live` job (schedule/manual/tag only, `-p x-media` since every network/secret-gated test lives there, `continue-on-error`) for the `#[ignore]`d live + token tests. `.github/workflows/docker.yml` builds and pushes the image on master/tag and runs a **build-only check on pull requests touching the build inputs** (`Dockerfile`, entrypoint, manifests, `.dockerignore`); a release tag must match both crate versions or the build stops, and `FFMPEG_URL`/`FFMPEG_SHA256` are taken from repository variables when set (a release can pin an exact ffmpeg build). `.github/dependabot.yml` keeps crates, the pinned actions and the Docker base images current. - Untested and hard to test without a mock seam: `main.rs`, `config.rs`, `db.rs`, `handlers/statics.rs`, `media_sender.rs` (holds the `MockSender` itself); in `x-media`: `media.rs`, `lib.rs`, all `model.rs`. The `commands.rs` *executor* needs a real `Bot` (only its pure report builder is tested). Everything else — `handlers/{mod,callback,inline,urls}.rs`, `send/*`, `ctx.rs`, `state.rs`, `queue.rs`, `link_cache.rs`, `rate_limit.rs` — is driven through `TestStores`/`ctx::test_support` and the scripted `MockSender`. - No coverage tracking. diff --git a/Dockerfile b/Dockerfile index 271ba58..650c0c9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,6 +11,8 @@ ARG APP_NAME=telegram-twitter-media-bot # runners. `/redirect/latest/` floats to the newest release build; each build # also ships a .sha256. Swap `amd64` for `arm64` when building arm64 images. ARG FFMPEG_URL=https://ffmpeg.martin-riedl.de/redirect/latest/linux/amd64/release/ffmpeg.zip +# Arm64 images need this URL swapped for the `linux/arm64` build (currently +# hardcoded amd64; the workflow builds amd64 only — see docker.yml). # Optional sha256 of ffmpeg.zip (pinned releases only): set to verify the # download. The mirror publishes .sha256 sidecars next to pinned builds, e.g. # https://ffmpeg.martin-riedl.de/download/linux/amd64/_9.0/ffmpeg.zip.sha256 @@ -28,7 +30,7 @@ COPY crates/xmedia-bot/Cargo.toml crates/xmedia-bot/Cargo.toml RUN mkdir -p crates/x-media/src crates/xmedia-bot/src \ && printf 'fn main() {}\n' > crates/xmedia-bot/src/main.rs \ && : > crates/x-media/src/lib.rs \ - && cargo build --release -p xmedia-bot + && cargo build --release --locked -p xmedia-bot # 2. Static ffmpeg next (cached unless FFMPEG_URL changes), so source edits # never re-download it. The zip contains a single `ffmpeg` binary at the @@ -51,7 +53,7 @@ RUN wget -q -O /tmp/ffmpeg.zip "$FFMPEG_URL" \ # removes 0 files and the stub binary silently ships.) COPY crates/ ./crates/ RUN find crates -type f -name '*.rs' -exec touch {} + \ - && cargo build --release -p xmedia-bot + && cargo build --release --locked -p xmedia-bot # ---------- runtime stage ---------- FROM debian:bookworm-slim diff --git a/crates/xmedia-bot/Cargo.toml b/crates/xmedia-bot/Cargo.toml index 964905d..2f88eb2 100644 --- a/crates/xmedia-bot/Cargo.toml +++ b/crates/xmedia-bot/Cargo.toml @@ -5,7 +5,7 @@ edition = "2024" [dependencies] teloxide = { version = "0.17", default-features = false, features = ["webhooks-axum", "macros", "rustls", "ctrlc_handler"] } -tokio = { version = "1.40", features = ["rt-multi-thread", "macros", "time"] } +tokio = { version = "1.40", features = ["rt-multi-thread", "macros", "time", "sync"] } serde = { version = "1", features = ["derive"] } serde_json = "1" log = "0.4"