From 0eb4e5c78d3a514036d5b46ad26f7bcd4067617e Mon Sep 17 00:00:00 2001 From: YoursFunny Date: Thu, 17 Sep 2026 22:34:11 +0800 Subject: [PATCH] fix(sites): request the opus serialization so bilibili posts keep their text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An image/text post fetched without `features=itemOpusStyle` comes back in bilibili's legacy shape, where the post's body and headline are gone completely — `desc: null`, no `major.opus` — so `title` (and `{title}`) stayed empty for exactly the posts that do have content (`opus/1248857553488576532`: legacy `desc: null`, flagged `major.opus.summary.text = "[doge_金箍]黑白搭配"`). The same flag also moves the pictures to `major.opus.pics` (key `url`, not `src`). Text now falls back opus (headline + body) → `desc.text` → archive card title; media falls back `opus.pics` → `draw.items` → archive cover, so the legacy shapes keep working if the flag is ever retired. Verified live: the reported link now yields title "[doge_金箍]黑白搭配" with its picture; AV dynamics keep their card title; forwards keep `desc.text` and gain `//@` composition unchanged. --- AGENTS.md | 2 +- BILIBILI_PLAN.md | 11 +- crates/x-media/src/site/bilibili/interface.rs | 203 +++++++++++++++--- crates/x-media/src/site/bilibili/model.rs | 40 +++- 4 files changed, 217 insertions(+), 39 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 25bdfca..0252ec5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,7 +31,7 @@ The `x-media` library: `site::fetch(url)` dispatches through the `SITES` registr | Path | Purpose | |---|---| | `crates/x-media/src/` | Fetch library. `site/mod.rs` = dispatcher + `Fetched`/`FetchError`/`download_media`/`media_size`; `media.rs` = `Media` enum; `examples/fetch.rs` = end-to-end usage sample | -| `crates/x-media/src/site//` | One directory per site: `mod.rs` (re-exports), `interface.rs` (PATTERN, `enabled()`, `fetch_from_url()`, `cache_key`/`is_retryable`/`media_headers`, unit struct `Site` implementing `site::Site`, `From for Fetched`), `model.rs` (serde DTOs). Pixiv adds `api.rs` (auth + transport); twitter adds `auth.rs` (logged-in GraphQL `TweetDetail` fallback for NSFW tweets, gated on `TWITTER_AUTH_TOKEN`). Misskey targets misskey.io only (`POST /api/notes/show`, 400+`NO_SUCH_NOTE` → NotFound). Bilibili fetches dynamics (images/animated images only — an attached video degrades to its cover, and its title stands in for the post text, which AV dynamics do not have) from the anonymous `/x/polymer/web-dynamic/v1/detail` (no WBI signature; device cookies `buvid3`/`buvid4` are fetched automatically from `/x/frontend/finger/spi` because bilibili's `-352` risk control starts rejecting plain requests, `BILIBILI_COOKIE` is the escalation when an IP stays blocked; `b23.tv` short links are deliberately unmatched). Twitter's `from_syndication_json` HTML-decodes the API text — syndication and GraphQL `full_text` both arrive pre-escaped (`>` `<` `&` `'`) — so the stored text is raw and the caption escap… +| `crates/x-media/src/site//` | One directory per site: `mod.rs` (re-exports), `interface.rs` (PATTERN, `enabled()`, `fetch_from_url()`, `cache_key`/`is_retryable`/`media_headers`, unit struct `Site` implementing `site::Site`, `From for Fetched`), `model.rs` (serde DTOs). Pixiv adds `api.rs` (auth + transport); twitter adds `auth.rs` (logged-in GraphQL `TweetDetail` fallback for NSFW tweets, gated on `TWITTER_AUTH_TOKEN`). Misskey targets misskey.io only (`POST /api/notes/show`, 400+`NO_SUCH_NOTE` → NotFound). Bilibili fetches dynamics (images/animated images only — an attached video degrades to its cover, and its title stands in for the post text, which AV dynamics do not have) from `/x/polymer/web-dynamic/v1/detail` sent with `features=itemOpusStyle` (without that flag the legacy serialization drops an image/text post's body and headline entirely — `desc` comes back `null`; the adapter still parses the legacy `major.draw`/`desc`/`archive` shapes as a fallback). No WBI signature is involved; device cookies `buvid3`/`buvid4` are fetched automatically from `/x/frontend/finger/spi` because bilibili's `-352` risk control starts rejecting plain requests, `BILIBILI_COOKIE` is the escalation when an IP stays blocked; `b23.tv` short links are deliberately unmatched. Twitter's `from_syndication_json` HTML-decodes the API text — syndication and GraphQL `full_text` both arrive pre-escaped (`>` `<` `&` `'`) — so the stored text is raw and the caption escap… | `crates/xmedia-bot/src/main.rs` | Entry point: env/log init, command registration (`register_commands`), shared `send::BOT` force-init, queue worker start, site login validation (`site::validate_all`), 300 s edit-expiry sweep, dptree handler tree, webhook vs polling dispatch | | `crates/xmedia-bot/src/config.rs` | Manual env parsing into `Config` | | `crates/xmedia-bot/src/db.rs` | `DbPool`: one shared SQLite connection pool (`POOL_SIZE = 4`, WAL, busy_timeout) for all three tables over `$DATA_DIR/task_queue.db` (default `data/`) — the three stores share it; `open_store` creates file + schema, `with_conn` runs all rusqlite I/O in `spawn_blocking` | diff --git a/BILIBILI_PLAN.md b/BILIBILI_PLAN.md index 09dad91..4cdaf33 100644 --- a/BILIBILI_PLAN.md +++ b/BILIBILI_PLAN.md @@ -69,6 +69,7 @@ | `dyn_archive` 字段 | 有 `aid/bvid/cover/title/duration_text`，**没有 `cid`**（所以发流要再来一次 `view` 请求） | | **风控阶梯（同一 IP 连续请求后实测）** | ① 无 cookie → `-352`；② 仅 `buvid3` → 仍 `-352`；③ `buvid3`+`buvid4`（取自匿名 `/x/frontend/finger/spi`）→ **`code:0` 恢复**；④ 继续高频请求后 → 连同 buvid 一起 `-352`（此时只有登录 cookie 或换 IP） | | **正文位置（24 条真实动态逐条审计）** | 有正文的动态都在 `module_dynamic.desc.text`（图文/转发/纯文字，含 34–193 字样本）；**AV（视频投稿）动态 `desc` 恒为 `null`**，内容在 `major.archive.title` / `.desc` 卡片里 → 已做 title 回退 | +| **`features=itemOpusStyle` 的效果** | 同一端点带此参数后，图文帖改为 `major.opus` 形态：`pics[]`（图，key 是 `url`）、`summary.text`（正文，未截断，实测 307 字整段）、`title`（可选标题）；不带参数则是 legacy `major.draw` + `desc`，而 **opus 图文帖的 `desc` 为 `null`、正文与标题完全丢失**（`opus/1248857553488576532`：legacy `desc:null`，带参数 `summary.text="[doge_金箍]黑白搭配"`）。AV / 转发帖不受该参数影响 → 适配器改为请求时带参数，并保留 legacy 形态兜底 | | feed 与 detail 的差异 | `feed/space` 的 item 会把 `desc.text` 挖空，**只有 detail 有正文** → 排查时不要用 feed 数据判断正文缺失 | | 不存在的 19 位 id | `4101105 请求数据发生错误`（提示可重试，但只出现在不可能存在的 id 上）→ 仍归入永久错误，见 `code_error` 注释 | @@ -114,15 +115,16 @@ crates/x-media/src/site/bilibili/model.rs # 纯 Deserialize DTO（全 Optio - **错误映射**：`0` → 成功；`-352/-412` 与 HTTP 412 → `Transient`（可重试，队列退避；首次记一条 warn 提示 `BILIBILI_COOKIE`）；`500`/`4101147` → `NotFound`（永久）；其他 code → `Site`（永久）。 - **媒体**： - - `major.draw.items[]` → 每张一张图（`http://` / `//` → `https://`，非 https 开头直接丢弃）； + - `major.opus.pics[]`（带 `features=itemOpusStyle` 时的图文帖形态，字段名是 `url`）→ 每张一张图； + 其次 `major.draw.items[]`（legacy，字段名 `src`）→ 同样逐张；`http://` / `//` → `https://`，非 https 开头直接丢弃。 `.gif` → `Media::Animated`（`thumbnail_url` 留空，Telegram 自己取首帧——`@518w.jpg` 只对 jpg/webp 实测过）， 其余 → `Media::Illustration`（`thumbnail_url = url + "@518w.jpg"`，兼作超大时的降级 URL）。 - `major.archive.cover` → 1 张 `Illustration`（视频不发流）。 - 转发且自身无媒体 → 递归取 `orig` 的媒体；正文拼 `//@{原作者}:\n{原文}`。 - 其他 major（PGC/ARTICLE/MUSIC/LIVE/COMMON）不建模 → 无媒体，走既有 "No media found"。 -- **正文 / title**：`module_dynamic.desc.text`；为空时回退到 **`major.archive.title`**。 - 实测（24 条真实动态审计 + 9 条 AV 动态）AV 动态（视频投稿）的 `desc` 恒为 `null`——它的"内容"就是卡片， - 不回退则所有视频动态的 `title`/`{title}` 都是空的。有正文的动态（图文/转发/纯文字）`desc.text` 实测正常。 +- **正文 / title**（按信息量从多到少回退）：`major.opus.title` + `major.opus.summary.text` + → `module_dynamic.desc.text` → `major.archive.title`。三者分别对应：图文文档（标题+正文）、 + legacy/转发帖正文、视频投稿卡片标题。开头结尾空白做 trim；整体再由既有 `truncate_caption` 截断。 - **caption**（与 misskey 同形）：`{opus 链接}\n{name}: {正文}`； `RenderData` 的 `{tags}` 来自话题名；正文由既有 `truncate_caption` 截断。 - **注册表**：`SITES` 末尾追加 → `/set_format` 白名单、链接缓存、启动校验、日志前缀全部自动生效。 @@ -138,6 +140,7 @@ crates/x-media/src/site/bilibili/model.rs # 纯 Deserialize DTO（全 Optio | `validate()` 校验 cookie | 不做 | 匿名可用，cookie 失效不致命；校验要额外请求一个端点，收益低 | | `media_headers` 给 hdslb 加 Referer | 返回 `None` | 实测图片与 durl 均无需 Referer（注释里记了这条验证） | | 计划阶段认为设备 cookie 是 YAGNI，不实现 | **实现**（`buvid3`+`buvid4`） | 计划之后做了对照实验：同一 IP 上"无 cookie → -352、只有 buvid3 → -352、buvid3+buvid4 → code:0"，说明这是对本适配器主要失败模式的直接修复，而不是冗余保险 | +| 只用不带参数的 `v1/detail` | 加 `features=itemOpusStyle` | 用户实测反馈"有内容的动态没有 title"：不带参数时 opus 图文帖返回 legacy 形态，`desc` 为 `null`，正文与标题整个丢失。带参数后同一 ID 返回 `major.opus.summary.text` / `title` / `pics`。AV / 转发帖不受影响，legacy 形态仍保留为兜底 | --- diff --git a/crates/x-media/src/site/bilibili/interface.rs b/crates/x-media/src/site/bilibili/interface.rs index ab88991..9e19e1a 100644 --- a/crates/x-media/src/site/bilibili/interface.rs +++ b/crates/x-media/src/site/bilibili/interface.rs @@ -9,8 +9,10 @@ //! `x/player/playurl` and its size/quality chasing — the cover plus the post //! link is what the operator asked for. //! -//! Text: a post's own body (`module_dynamic.desc.text`); for a 视频投稿动态 -//! the body is empty by construction, so the archive card's title stands in. +//! Text: an image/text post is read from its opus serialization +//! (`features=itemOpusStyle` — the legacy serialization carries no text at +//! all), a forward from its own body plus the quote's, and a 视频投稿动态 +//! from the archive card's title, which it has instead of a body. //! //! `b23.tv` short links are not matched: most of them point at videos, which //! this adapter does not handle, and matching them would turn a silently @@ -42,6 +44,17 @@ const SPI_URL: &str = "https://api.bilibili.com/x/frontend/finger/spi"; /// risk-controls (`code -352`, HTTP 412). const REFERER: &str = "https://www.bilibili.com/"; +/// Web feature flag that makes the detail endpoint serialize an image/text +/// post as `major.opus` instead of the legacy `major.draw` + `desc` pair. +/// +/// Not optional: without it an opus post comes back with **no text at all** +/// (`desc: null` and no `major.opus`), which is how the document title and +/// body — the post's actual content — used to be lost (verified 2026-09-17 +/// on `opus/1248857553488576532`: legacy `desc: null`, flagged +/// `major.opus.summary.text = "[doge_金箍]黑白搭配"`). The adapter still +/// parses the legacy shape as a fallback, in case the flag is retired. +const OPUS_FEATURE: &str = "itemOpusStyle"; + /// hdslb image variant used as thumbnail (and as the oversized fallback): a /// downscaled still of the same image, ~30 KB instead of ~570 KB. Verified /// live for `.jpg` and `.webp` sources. Not applied to `.gif` (unverified for @@ -194,7 +207,7 @@ async fn request(url: &str) -> reqwest::RequestBuilder { pub async fn fetch(dynamic_id: &str) -> Result { let response = request(API_URL) .await - .query(&[("id", dynamic_id)]) + .query(&[("id", dynamic_id), ("features", OPUS_FEATURE)]) .send() .await?; let status = response.status(); @@ -307,32 +320,55 @@ fn desc_text(item: &model::Item) -> &str { .unwrap_or_default() } -fn archive_title(item: &model::Item) -> Option<&str> { +fn major(item: &model::Item) -> Option<&model::Major> { item.modules .as_ref() .and_then(|modules| modules.module_dynamic.as_ref()) .and_then(|dynamic| dynamic.major.as_ref()) +} + +fn opus(item: &model::Item) -> Option<&model::Opus> { + major(item).and_then(|major| major.opus.as_ref()) +} + +fn archive_title(item: &model::Item) -> Option<&str> { + major(item) .and_then(|major| major.archive.as_ref()) .and_then(|archive| archive.title.as_deref()) .filter(|title| !title.trim().is_empty()) } -/// The dynamic's own words: its body, or the attached video's title when the -/// body is empty. +/// The dynamic's own words, richest source first: the opus document +/// (headline plus body) → `module_dynamic.desc.text` → the attached video's +/// card title. /// -/// A 视频投稿动态 (`MAJOR_TYPE_ARCHIVE`) carries **no body at all** — `desc` -/// comes back `null`, the content being the archive card (verified on 9 live -/// AV dynamics 2026-09-17). Falling back to the card title is what keeps -/// `title` (and `{title}` in caption formats) populated for the most common -/// dynamic type, mirroring pixiv, whose `title` is the artwork title rather -/// than post text. -fn own_text(item: &model::Item) -> &str { - let body = desc_text(item); - if body.trim().is_empty() { - archive_title(item).unwrap_or_default() - } else { - body +/// The opus shape is what makes ordinary 图文 posts readable at all — their +/// legacy serialization has no text — while a 视频投稿动态 has no body +/// anywhere and is represented by its card title (mirroring pixiv, whose +/// `title` is the artwork title rather than post text). +fn own_text(item: &model::Item) -> String { + if let Some(opus) = opus(item) { + let title = opus.title.as_deref().unwrap_or_default().trim(); + let body = opus + .summary + .as_ref() + .map(|summary| summary.text.trim()) + .unwrap_or_default(); + let text = match (title.is_empty(), body.is_empty()) { + (false, false) => format!("{title}\n{body}"), + (false, true) => title.to_string(), + (true, false) => body.to_string(), + (true, true) => String::new(), + }; + if !text.is_empty() { + return text; + } } + let body = desc_text(item).trim(); + if !body.is_empty() { + return body.to_string(); + } + archive_title(item).unwrap_or_default().to_string() } fn topic_name(item: &model::Item) -> &str { @@ -349,19 +385,19 @@ fn topic_name(item: &model::Item) -> &str { fn text_of(item: &model::Item) -> String { let own = own_text(item); let Some(orig) = item.orig.as_deref() else { - return own.to_string(); + return own; }; let orig_text = own_text(orig); if orig_text.is_empty() { - return own.to_string(); + return own; } let name = author_name(orig); - let mut text = own.to_string(); + let mut text = own; if !text.is_empty() { text.push('\n'); } if name.is_empty() { - text.push_str(orig_text); + text.push_str(&orig_text); } else { text.push_str(&format!("//@{name}:\n{orig_text}")); } @@ -380,19 +416,26 @@ fn media_of(item: &model::Item) -> Vec { } fn own_media(item: &model::Item) -> Vec { - let Some(major) = item - .modules - .as_ref() - .and_then(|modules| modules.module_dynamic.as_ref()) - .and_then(|dynamic| dynamic.major.as_ref()) - else { + // Flagged serialization first: image posts put their pictures in + // `major.opus.pics` (and carry no `major.draw` at all). + if let Some(opus) = opus(item) { + let pics: Vec = opus + .pics + .iter() + .filter_map(|pic| pic.url().and_then(image)) + .collect(); + if !pics.is_empty() { + return pics; + } + } + let Some(major) = major(item) else { return Vec::new(); }; if let Some(draw) = major.draw.as_ref() { return draw .items .iter() - .filter_map(|pic| image(&pic.src)) + .filter_map(|pic| pic.url().and_then(image)) .collect(); } major @@ -628,6 +671,91 @@ mod tests { assert!(fetched.media.is_empty()); } + /// The serialization the web client asks for: an image/text post arrives + /// as `major.opus` (pictures, body and optional headline) with **no** + /// `desc` — the legacy shape would lose the text entirely. + #[test] + fn from_item_opus_shape_carries_text_and_pics() { + let major = serde_json::json!({ + "type": "MAJOR_TYPE_OPUS", + "opus": { + "title": "每个人的青春里，都有一首 A-Lin", + "summary": { "text": "那些曾经陪你失恋的歌\n\n【活动】详情见正文" }, + "pics": [ + { "url": "http://i0.hdslb.com/bfs/new_dyn/a.jpg", "width": 2304, "height": 2880 }, + { "url": "http://i0.hdslb.com/bfs/new_dyn/b.jpg", "width": 2304, "height": 2880 }, + ], + "fold_action": ["展开", "收起"], + "jump_url": "//www.bilibili.com/opus/1245284537985925159", + }, + }); + let item = { + let mut json = item_json(major, ""); + json["modules"]["module_dynamic"]["desc"] = serde_json::Value::Null; + json + }; + let fetched = parse(item); + + assert_eq!( + fetched.title, + "每个人的青春里，都有一首 A-Lin\n那些曾经陪你失恋的歌\n\n【活动】详情见正文" + ); + assert!( + fetched.caption.contains("那些曾经陪你失恋的歌"), + "{}", + fetched.caption + ); + assert_eq!(fetched.media.len(), 2); + match &fetched.media[0] { + Media::Illustration { + url, thumbnail_url, .. + } => { + assert_eq!(url, "https://i0.hdslb.com/bfs/new_dyn/a.jpg"); + assert_eq!( + thumbnail_url.as_deref(), + Some("https://i0.hdslb.com/bfs/new_dyn/a.jpg@518w.jpg") + ); + } + other => panic!("{other:?}"), + } + assert_eq!( + fetched.media[1].url(), + "https://i0.hdslb.com/bfs/new_dyn/b.jpg" + ); + + // A body without a headline, and a headline without a body, both + // stand alone rather than rendering an empty line. + for (title, body, expected) in [ + (None, "只有正文", "只有正文"), + (Some("只有标题"), "", "只有标题"), + ] { + let major = serde_json::json!({ + "type": "MAJOR_TYPE_OPUS", + "opus": { "title": title, "summary": { "text": body }, "pics": [] }, + }); + let mut json = item_json(major, ""); + json["modules"]["module_dynamic"]["desc"] = serde_json::Value::Null; + assert_eq!(parse(json).title, expected); + } + } + + /// The legacy shape stays supported: bilibili's `itemOpusStyle` flag is + /// what moves the pictures to `major.opus.pics`, but `major.draw` items + /// and a text-only `desc` must keep working if it is retired. + #[test] + fn from_item_legacy_draw_shape_still_parses() { + let fetched = parse(item_json( + draw_item("http://i0.hdslb.com/bfs/new_dyn/l.jpg"), + "legacy 正文", + )); + assert_eq!(fetched.title, "legacy 正文"); + assert_eq!(fetched.media.len(), 1); + assert_eq!( + fetched.media[0].url(), + "https://i0.hdslb.com/bfs/new_dyn/l.jpg" + ); + } + /// The video stream is out of scope; an AV dynamic still yields its cover. #[test] fn from_item_maps_archive_cover() { @@ -855,6 +983,23 @@ mod tests { assert!(!fetched.title.trim().is_empty()); } + /// Regression for the reported case: this opus post's legacy + /// serialization has `desc: null`, so without the `itemOpusStyle` + /// request it parsed with an empty title. + #[tokio::test] + #[ignore = "live network: requires outbound HTTPS to api.bilibili.com"] + async fn live_fetch_opus_dynamic_has_title_and_text() { + let Some(fetched) = live_fetch("https://www.bilibili.com/opus/1248857553488576532").await + else { + return; + }; + assert_eq!(fetched.title, "[doge_金箍]黑白搭配"); + assert!(fetched.caption.ends_with("黑白搭配"), "{}", fetched.caption); + let urls: Vec<&str> = fetched.media.iter().map(|m| m.url()).collect(); + assert_eq!(urls.len(), 1, "{urls:?}"); + assert!(urls[0].starts_with("https://i0.hdslb.com/"), "{urls:?}"); + } + #[tokio::test] #[ignore = "live network: requires outbound HTTPS to api.bilibili.com"] async fn live_fetch_video_dynamic_uses_archive_title() { diff --git a/crates/x-media/src/site/bilibili/model.rs b/crates/x-media/src/site/bilibili/model.rs index 9a30118..6a13107 100644 --- a/crates/x-media/src/site/bilibili/model.rs +++ b/crates/x-media/src/site/bilibili/model.rs @@ -68,14 +68,16 @@ pub(crate) struct Desc { pub(crate) text: String, } -/// `major` is a tagged union: `type` (`MAJOR_TYPE_DRAW` / `_ARCHIVE` / …) -/// plus one payload object per type. Only the two payloads this adapter reads -/// are modeled; an unknown major simply yields no media. +/// `major` is a tagged union: `type` (`MAJOR_TYPE_DRAW` / `_OPUS` / +/// `_ARCHIVE` / …) plus one payload object per type. Only the three payloads +/// this adapter reads are modeled; an unknown major simply yields no media. #[derive(Deserialize, Debug)] pub(crate) struct Major { #[serde(default)] pub(crate) draw: Option, #[serde(default)] + pub(crate) opus: Option, + #[serde(default)] pub(crate) archive: Option, } @@ -87,8 +89,36 @@ pub(crate) struct Draw { #[derive(Deserialize, Debug)] pub(crate) struct Pic { - /// Image URL, served as `http://` — normalized to https by the adapter. - pub(crate) src: String, + /// `major.draw` image URL. + #[serde(default)] + pub(crate) src: Option, + /// `major.opus.pics` image URL — the opus shape names the field + /// differently while carrying the same image. + #[serde(default)] + pub(crate) url: Option, +} + +impl Pic { + /// The image URL, whichever key this serialization put it under. + pub(crate) fn url(&self) -> Option<&str> { + self.src.as_deref().or(self.url.as_deref()) + } +} + +/// `major.opus`: the serialization of an image/text post the web client asks +/// for (`features=itemOpusStyle`). It carries the parts the legacy shape drops +/// entirely — the document title and body of an opus post, whose +/// `module_dynamic.desc` comes back `null`. +#[derive(Deserialize, Debug)] +pub(crate) struct Opus { + /// Document headline; often absent. + #[serde(default)] + pub(crate) title: Option, + /// Document body (untruncated: a 307-char sample came back whole). + #[serde(default)] + pub(crate) summary: Option, + #[serde(default)] + pub(crate) pics: Vec, } #[derive(Deserialize, Debug)]