From 994734b001df1ac100d6956ad3ef793d55a20b85 Mon Sep 17 00:00:00 2001 From: YoursFunny Date: Mon, 21 Sep 2026 13:09:32 +0800 Subject: [PATCH] perf: gzip and HTTP/2 for the site clients MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit x-media's reqwest had exactly `["json", "rustls-tls"]` — no decompression, no HTTP/2 — so every adapter fetched its JSON as identity over HTTP/1.1. The site APIs compress: twitter's syndication body measures 4469 bytes identity against 1066 gzipped (4.2x) for a single-tweet response, and the fetch is on every twitter link. The CDNs all negotiate h2 (the shared client reports HTTP/2.0 against cdn.syndication.twimg.com, api.bilibili.com and public.api.bsky.app), which also multiplexes the concurrent media downloads that used to open a connection each. Both features are one crate-level switch, so the pages' compression is what the APIs answer with and requires no adapter change: reqwest adds `accept-encoding: gzip` and decompresses transparently. `download_media_limited` keeps a correct size cap either way — it checks the accumulated *body* while streaming, not the declared Content-Length, which for a compressed response is the compressed size. Verified: a local echo server recorded `accept-encoding: gzip` on a request from the shared client (both clients come from `build_client`), and a probe over the shared `CLIENT` reported `HTTP/2.0` for the three site hosts above. `cargo test -p x-media -- --ignored live` (14 passed) covers the real endpoints with the new transport. Cargo.lock gains async-compression (+codecs/core), fnv and h2. `cargo fmt --check`, `cargo clippy --workspace --all-targets --locked -- -D warnings` and `cargo test --workspace --locked` clean. --- AGENTS.md | 2 +- Cargo.lock | 61 +++++++++++++++++++++++++++++++++++++++ crates/x-media/Cargo.toml | 2 +- 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1d8919d..f54da33 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -96,7 +96,7 @@ Docker: `docker build -t tgxmb .` then `docker run --rm -d --name tgxmb --env-fi - **Rust, stable, edition 2024**, workspace resolver 3. No `rust-version`/MSRV pin, no `rust-toolchain.toml` — recent stable is assumed. No nightly features. - Package manager: **Cargo** (workspace with path dep `x-media` ← `xmedia-bot`). No `[workspace.package]`/shared deps — each crate lists deps independently. -- **TLS is rustls end-to-end** (no native-tls/openssl in the tree, no libssl in the Docker runtime image): `teloxide` is declared `default-features = false` with `["webhooks-axum", "macros", "rustls", "ctrlc_handler"]` (the removed `default` also carried `native-tls` and `ctrlc_handler` — the latter must stay); x-media's reqwest is `default-features = false` with `["json", "rustls-tls"]` (webpki-roots baked in, so the image ships no CA bundle). One reqwest 0.12.28 in the lock. +- **TLS is rustls end-to-end** (no native-tls/openssl in the tree, no libssl in the Docker runtime image): `teloxide` is declared `default-features = false` with `["webhooks-axum", "macros", "rustls", "ctrlc_handler"]` (the removed `default` also carried `native-tls` and `ctrlc_handler` — the latter must stay); x-media's reqwest is `default-features = false` with `["json", "rustls-tls", "gzip", "http2"]` (webpki-roots baked in, so the image ships no CA bundle; `gzip` because the site APIs answer their JSON compressed — twitter's syndication body is 4469 bytes identity vs 1066 gzipped — and `http2` because every site CDN here negotiates h2). One reqwest 0.12.28 in the lock. - **Versioning**: bump the version in all three places (`crates/x-media/Cargo.toml`, `crates/xmedia-bot/Cargo.toml`, `Cargo.lock`) and **keep `README.md`, `README.en.md` and `AGENTS.md` in sync with the code on every bump**, then commit (`chore: bump version to X.Y.Z`), create an annotated tag `vX.Y.Z`, and push branch + tag (the tag push triggers the Docker Hub build). The tag must equal both crate versions: `.github/workflows/docker.yml` verifies that before building, and `--locked` verifies the lock file. - Config is **environment-variable driven** (dotenv loads `.env`, which is gitignored; `.env.example` is the tracked template — `cp .env.example .env` — and is also the file `docker compose` substitutes `${VAR}` from, so every variable the compose passes must be documented there). Key vars: `TELOXIDE_TOKEN` (required), `PIXIV_REFRESH_TOKEN`, `TWITTER_AUTH_TOKEN` (optional; x.com `auth_token` cookie — enables the logged-in GraphQL fallback that fetches NSFW tweets syndication withholds), `BILIBILI_COOKIE` (optional; whole bilibili cookie string — bilibili dynamics fetch anonymously and add their own device cookies, this only rescues an egress IP that bilibili has hard-flagged with `-352`/412), `BOT_ADMIN` (comma-separated ids), `EDIT_MESSAGE_TTL_SECONDS` (default 86400), `LINK_CACHE_TTL_SECONDS` (default 604800), `CAPTION_QUOTE_TEXT_CHARS` (default 200; a post whose text — the `title` plus `content` joined, see `site::compose_text` — reaches this length gets that text wrapped in an expandable blockquote inside its caption, the URL and author line staying outside; `0` disables it. Applied at the send boundary in `send::quote_long_caption`, which locates the text as what follows the author link, so a `/set_format` that moves `{title}`/`{content}` elsewhere and pixiv's title-inside-a-link layout opt out; `copy_messages` forwards and queued retries inherit the wrap, while the edit-before-forward rewrite stays unquoted by design), `DATA_DIR` (default `data`, CWD-relative; the SQLite dir, auto-created), `WEBHOOK`/`WEBHOOK_URL`/`WEBHOOK_LISTEN`/`WEBHOOK_PORT`/`WEBHOOK_CERT`/`WEBHOOK_SECRET_TOKEN` (webhook mode requires URL/listen/port, `.expect`ed; `WEBHOOK_CERT` is Telegram-facing self-signed validation only — TLS must be terminated by a reverse proxy), `RUST_LOG`, `TELOXIDE_PROXY`, `LOCAL_USER_ID` (entrypoint only). - SQLite via `rusqlite` with `bundled` feature (no system libsqlite needed). DB file `$DATA_DIR/task_queue.db` (default `data/task_queue.db`, CWD-relative — run from the workspace root, or `/app` in Docker; set `DATA_DIR` to pin state anywhere). Mount `./data` and `./cert` volumes. diff --git a/Cargo.lock b/Cargo.lock index f10a841..2e648b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -60,6 +60,18 @@ dependencies = [ "object", ] +[[package]] +name = "async-compression" +version = "0.4.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3976abdc8fe7d1133d43d304afd42abdf5bc3e1319d263d223bde07b5efc4be8" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -266,6 +278,23 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + [[package]] name = "const-oid" version = "0.10.2" @@ -583,6 +612,12 @@ dependencies = [ "zlib-rs", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.2.0" @@ -713,6 +748,25 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "h2" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap 2.14.2", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -849,6 +903,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", + "h2", "http", "http-body", "httparse", @@ -1740,6 +1795,7 @@ dependencies = [ "bytes", "futures-core", "futures-util", + "h2", "http", "http-body", "http-body-util", @@ -2425,12 +2481,17 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ + "async-compression", "bitflags 2.13.2", "bytes", + "futures-core", "futures-util", "http", "http-body", + "http-body-util", "pin-project-lite", + "tokio", + "tokio-util", "tower", "tower-layer", "tower-service", diff --git a/crates/x-media/Cargo.toml b/crates/x-media/Cargo.toml index c39aa6f..8d32333 100644 --- a/crates/x-media/Cargo.toml +++ b/crates/x-media/Cargo.toml @@ -4,7 +4,7 @@ version = "1.8.0" edition = "2024" [dependencies] -reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls", "gzip", "http2"] } serde = { version = "1", features = ["derive"] } serde_json = "1" regex = "1.12"