diff --git a/crates/x-media/src/site/bilibili/interface.rs b/crates/x-media/src/site/bilibili/interface.rs index a0a4153..7f7ea2d 100644 --- a/crates/x-media/src/site/bilibili/interface.rs +++ b/crates/x-media/src/site/bilibili/interface.rs @@ -174,10 +174,7 @@ async fn cookie() -> Option { /// requests go out without a cookie. async fn fetch_buvid() -> Result, FetchError> { let response = crate::site::CLIENT.get(SPI_URL).send().await?; - let fingerprint: model::Fingerprint = response.json().await.map_err(|e| FetchError::Site { - site: "bilibili", - error: Box::new(e), - })?; + let fingerprint: model::Fingerprint = crate::site::response_json(response, "bilibili").await?; Ok(buvid_cookie(&fingerprint)) } @@ -213,17 +210,10 @@ pub async fn fetch(dynamic_id: &str) -> Result { if !status.is_success() { return Err(match status.as_u16() { 412 => risk_control("412"), - // Everything else shares the central classes (refusals and gone - // posts permanent, 429/5xx retried). The local fallback used to - // disagree: a bilibili 404 came back Transient here. 412 above is - // bilibili's risk control, which does clear on its own. _ => crate::site::status_error("bilibili", &response), }); } - let detail: model::Detail = response.json().await.map_err(|e| FetchError::Site { - site: "bilibili", - error: Box::new(e), - })?; + let detail: model::Detail = crate::site::response_json(response, "bilibili").await?; if let Some(err) = code_error(detail.code, detail.message.as_deref().unwrap_or_default()) { return Err(err); } diff --git a/crates/x-media/src/site/bsky/interface.rs b/crates/x-media/src/site/bsky/interface.rs index 051787a..00eef5e 100644 --- a/crates/x-media/src/site/bsky/interface.rs +++ b/crates/x-media/src/site/bsky/interface.rs @@ -329,7 +329,7 @@ pub async fn fetch(handle: &str, rkey: &str) -> Result { if !status.is_success() { return Err(crate::site::status_error("bsky", &response)); } - let text = response.text().await?; + let text = crate::site::response_text(response, "bsky").await?; Post::from_json(&text, rkey.to_string()) } diff --git a/crates/x-media/src/site/download.rs b/crates/x-media/src/site/download.rs index 96d62cf..bb77cae 100644 --- a/crates/x-media/src/site/download.rs +++ b/crates/x-media/src/site/download.rs @@ -133,6 +133,32 @@ async fn next_chunk(response: &mut reqwest::Response) -> Result Result { + if let Some(len) = response.content_length() + && len > crate::site::MAX_SITE_JSON_BYTES as u64 + { + return Err(FetchError::Site { + site, + error: "site response exceeds JSON size cap".into(), + }); + } + let mut body = Vec::new(); + while let Some(chunk) = next_chunk(&mut response).await? { + if body.len().saturating_add(chunk.len()) > crate::site::MAX_SITE_JSON_BYTES { + return Err(FetchError::Site { + site, + error: "site response exceeds JSON size cap".into(), + }); + } + body.extend_from_slice(&chunk); + } + Ok(bytes::Bytes::from(body)) +} + /// Whether an address must never be fetched. Media URLs come from a site's own /// API response and the bytes are uploaded to Telegram, so following one into /// the host's own network would turn the bot into a proxy for it: a cloud diff --git a/crates/x-media/src/site/misskey/interface.rs b/crates/x-media/src/site/misskey/interface.rs index b4e2641..626ba81 100644 --- a/crates/x-media/src/site/misskey/interface.rs +++ b/crates/x-media/src/site/misskey/interface.rs @@ -69,16 +69,13 @@ pub async fn fetch(note_id: &str) -> Result { _ => crate::site::status_error("misskey", &response), }); } - response.json().await.map_err(|e| FetchError::Site { - site: "misskey", - error: Box::new(e), - }) + crate::site::response_json(response, "misskey").await } /// Maps a 400 response: NO_SUCH_NOTE is permanent NotFound, any other 400 is /// a site error (permanent — retrying a rejected request cannot succeed). async fn not_found_or_invalid(response: reqwest::Response) -> FetchError { - match response.json::().await { + match crate::site::response_json::(response, "misskey").await { Ok(v) if v["error"]["code"] == "NO_SUCH_NOTE" => FetchError::NotFound, _ => FetchError::Site { site: "misskey", diff --git a/crates/x-media/src/site/mod.rs b/crates/x-media/src/site/mod.rs index 3518484..790bdec 100644 --- a/crates/x-media/src/site/mod.rs +++ b/crates/x-media/src/site/mod.rs @@ -224,6 +224,36 @@ pub fn caption_from_fields( ) } +/// Maximum decoded JSON response accepted from a site API. Metadata is +/// expected to be much smaller; this keeps a compromised or malformed API +/// from growing an unbounded `String` before serde gets a chance to reject it. +pub(crate) const MAX_SITE_JSON_BYTES: usize = 8 * 1024 * 1024; + +/// Reads a successful site response as a bounded UTF-8 JSON value. +pub(crate) async fn response_json( + response: reqwest::Response, + site: &'static str, +) -> Result { + let body = crate::site::download::send_json_response(response, site).await?; + serde_json::from_slice(&body).map_err(|e| FetchError::Site { + site, + error: Box::new(e), + }) +} + +/// Same bounded response reader for endpoints that need a text body before +/// classification or parsing. +pub(crate) async fn response_text( + response: reqwest::Response, + site: &'static str, +) -> Result { + let body = crate::site::download::send_json_response(response, site).await?; + String::from_utf8(body.to_vec()).map_err(|e| FetchError::Site { + site, + error: Box::new(e), + }) +} + /// Stable per-post cache key derived from any supported URL, so variant /// domains (x.com / twitter.com / fxtwitter.com, mobile, `/photo/N` /// suffixes) map to the same post. Delegates to each registered site's diff --git a/crates/x-media/src/site/pixiv/api.rs b/crates/x-media/src/site/pixiv/api.rs index de1280d..e3fd2c3 100644 --- a/crates/x-media/src/site/pixiv/api.rs +++ b/crates/x-media/src/site/pixiv/api.rs @@ -59,6 +59,15 @@ pub enum PixivError { Transient(String), } +fn map_response_error(error: FetchError) -> PixivError { + match error { + FetchError::Http(e) => PixivError::Http(e), + FetchError::Transient(message) => PixivError::Transient(message), + FetchError::RateLimited { .. } => PixivError::Transient("rate limited".into()), + other => PixivError::Api(other.to_string()), + } +} + /// Native pixiv app-API client. pub struct PixivAPI { refresh_token: String, @@ -94,14 +103,12 @@ impl PixivAPI { .header("User-Agent", AUTH_USER_AGENT) .send() .await?; - // Check the status *before* reading the body: a 429/5xx from the - // token endpoint is worth retrying (the class comes from - // `is_retryable`), while parsing a maintenance page as JSON turned it - // into a permanent `Api`/`Json` error with no retry at all. if !response.status().is_success() { return Err(PixivError::Status(response.status().as_u16())); } - let json: serde_json::Value = serde_json::from_str(&response.text().await?)?; + let json: serde_json::Value = crate::site::response_json(response, "pixiv") + .await + .map_err(map_response_error)?; let access_token = json .get("access_token") .and_then(|v| v.as_str()) @@ -140,7 +147,9 @@ impl PixivAPI { if !response.status().is_success() { return Err(PixivError::Status(response.status().as_u16())); } - let json: serde_json::Value = serde_json::from_str(&response.text().await?)?; + let json: serde_json::Value = crate::site::response_json(response, "pixiv") + .await + .map_err(map_response_error)?; if json.get("error").is_some() { let message = json .get("message") @@ -198,7 +207,9 @@ impl PixivAPI { if !response.status().is_success() { return Err(PixivError::Status(response.status().as_u16())); } - let json: serde_json::Value = serde_json::from_str(&response.text().await?)?; + let json: serde_json::Value = crate::site::response_json(response, "pixiv") + .await + .map_err(map_response_error)?; if json.get("error").is_some() { let message = json .get("message") @@ -484,6 +495,12 @@ mod tests { ); } + #[test] + fn response_read_transport_errors_stay_retryable() { + let mapped = map_response_error(FetchError::Transient("reset".into())); + assert!(matches!(mapped, PixivError::Transient(_))); + } + #[test] fn ugoira_budget_rejects_too_many_frames() { assert!(check_ugoira_archive_size(MAX_UGOIRA_FRAMES, 0).is_ok()); diff --git a/crates/x-media/src/site/twitter/auth.rs b/crates/x-media/src/site/twitter/auth.rs index 91beb24..4e7a455 100644 --- a/crates/x-media/src/site/twitter/auth.rs +++ b/crates/x-media/src/site/twitter/auth.rs @@ -132,7 +132,7 @@ pub async fn fetch(id: &str) -> Result { log::warn!("twitter auth fetch {id}: HTTP {status}"); return Err(crate::site::status_error("twitter auth", &response)); } - let text = response.text().await?; + let text = crate::site::response_text(response, "twitter auth").await?; let json: Value = serde_json::from_str(&text)?; let result = parse_tweet_result(&json, id)?; let syndication_shape = to_syndication_shape(&result).ok_or_else(|| { diff --git a/crates/x-media/src/site/twitter/interface.rs b/crates/x-media/src/site/twitter/interface.rs index 1ae0e5a..83c755a 100644 --- a/crates/x-media/src/site/twitter/interface.rs +++ b/crates/x-media/src/site/twitter/interface.rs @@ -89,7 +89,7 @@ pub async fn fetch(id: &str) -> Result { if !status.is_success() { return Err(crate::site::status_error("twitter", &response)); } - let text = response.text().await?; + let text = crate::site::response_text(response, "twitter").await?; // Classify before building the tweet (see [`parse_syndication_body`]), and // build it from the value that classification already parsed: this used to // scan and allocate the whole body twice.