refactor(x-media): drop the per-site fns the Site defaults already cover

twitter, bsky, misskey and bilibili each carried enabled() -> true,
media_headers(url) -> None and is_retryable(err) -> the trait's own default,
with no caller outside their tests (the adapters never override those
methods, so the default was already the production policy). The tests that
only restated the default are gone; the two that pin site-specific classes
(bsky's MediaPrep, bilibili's risk-control codes) now ask the Site impl, and
site/mod.rs keeps one assertion of the shared retry policy. The live
verification notes (no Referer needed for hdslb/twimg) survive as comments.
This commit is contained in:
2026-09-21 17:12:42 +08:00
parent 5630a86d88
commit 9a7cda05c9
9 changed files with 22 additions and 100 deletions
+5 -18
View File
@@ -111,10 +111,6 @@ pub static PATTERN: LazyLock<Regex> = LazyLock::new(|| {
.unwrap()
});
pub fn enabled() -> bool {
true
}
pub async fn fetch_from_url(url: &str) -> Result<Fetched, FetchError> {
let dynamic_id = PATTERN
.captures(url)
@@ -133,18 +129,9 @@ pub fn cache_key(url: &str) -> Option<String> {
.map(|caps| format!("bilibili:{}", &caps[1]))
}
/// Bilibili's fetch-retry policy: transient classes only. Not-found, blocked
/// and parse failures are permanent.
pub fn is_retryable(err: &FetchError) -> bool {
matches!(err, FetchError::Http(_) | FetchError::Transient(_))
}
/// hdslb media serves without a `Referer` (verified live 2026-09-17 on
/// `i0.hdslb.com` image URLs, requested both with and without one), so no
/// extra headers.
pub fn media_headers(_url: &str) -> Option<Vec<(&'static str, String)>> {
None
}
// hdslb media serves without a `Referer` (verified live 2026-09-17 on
// `i0.hdslb.com` image URLs, requested both with and without one), so this
// adapter does not override `Site::media_headers`.
/// `Cookie` header for bilibili requests: the operator's `BILIBILI_COOKIE`
/// when set, otherwise the anonymous device cookies.
@@ -908,7 +895,7 @@ mod tests {
// dropping the post.
for code in [-352, -412] {
let err = code_error(code, "-352").unwrap();
assert!(is_retryable(&err), "{err}");
assert!(BilibiliSite.is_retryable(&err), "{err}");
}
// A removed dynamic is permanent.
assert!(matches!(code_error(500, ""), Some(FetchError::NotFound)));
@@ -917,7 +904,7 @@ mod tests {
Some(FetchError::NotFound)
));
let err = code_error(-400, "param parsing failed").unwrap();
assert!(!is_retryable(&err), "{err}");
assert!(!BilibiliSite.is_retryable(&err), "{err}");
assert!(err.to_string().contains("-400"), "{err}");
}
+1 -3
View File
@@ -1,6 +1,4 @@
mod interface;
mod model;
pub use interface::{
BilibiliSite, PATTERN, cache_key, enabled, fetch_from_url, is_retryable, media_headers,
};
pub use interface::{BilibiliSite, PATTERN, cache_key, fetch_from_url};
+3 -17
View File
@@ -30,10 +30,6 @@ pub static PATTERN: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(r"^(?:https?://)?bsky\.app/profile/([\w.\-:]+)/post/([\w.\-~]+)").unwrap()
});
pub fn enabled() -> bool {
true
}
pub async fn fetch_from_url(url: &str) -> Result<Fetched, FetchError> {
let caps = PATTERN.captures(url).ok_or(FetchError::NotFound)?;
let handle = caps
@@ -112,17 +108,6 @@ pub fn cache_key(url: &str) -> Option<String> {
.map(|caps| format!("bsky:{}/{}", &caps[1], &caps[2]))
}
/// Bluesky's fetch-retry policy: transient classes only. Not-found, blocked
/// and parse failures are permanent.
pub fn is_retryable(err: &FetchError) -> bool {
matches!(err, FetchError::Http(_) | FetchError::Transient(_))
}
/// bsky media (cdn.bsky.app) needs no extra headers.
pub fn media_headers(_url: &str) -> Option<Vec<(&'static str, String)>> {
None
}
/// Segments fetched (and written) at once while remuxing an HLS video. Small
/// on purpose: a segment can be up to 20 MiB and the whole playlist is capped
/// at 256 MiB, so this is also what bounds the remux's peak memory.
@@ -499,10 +484,11 @@ mod tests {
/// ([`fetch_hls`]). The classes below are the ones still retried there.
#[test]
fn media_prep_failure_is_not_retried() {
assert!(!is_retryable(&FetchError::MediaPrep(
use crate::site::Site as _;
assert!(!BskySite.is_retryable(&FetchError::MediaPrep(
"bsky video remux failed: segment 400: 503".into()
)));
assert!(is_retryable(&FetchError::Transient("429".into())));
assert!(BskySite.is_retryable(&FetchError::Transient("429".into())));
}
#[test]
+1 -3
View File
@@ -1,6 +1,4 @@
mod interface;
mod model;
pub use interface::{
BskySite, PATTERN, Post, cache_key, enabled, fetch_from_url, is_retryable, media_headers,
};
pub use interface::{BskySite, PATTERN, Post, cache_key, fetch_from_url};
@@ -34,10 +34,6 @@ impl Site for MisskeySite {
pub static PATTERN: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"^(?:https?://)?misskey\.io/notes/([\w.\-~]+)").unwrap());
pub fn enabled() -> bool {
true
}
pub async fn fetch_from_url(url: &str) -> Result<Fetched, FetchError> {
let caps = PATTERN.captures(url).ok_or(FetchError::NotFound)?;
let note_id = caps.get(1).ok_or(FetchError::NotFound)?.as_str();
@@ -53,17 +49,6 @@ pub fn cache_key(url: &str) -> Option<String> {
.map(|caps| format!("misskey:{}", &caps[1]))
}
/// Misskey's fetch-retry policy: transient classes only. Not-found, blocked
/// and parse failures are permanent.
pub fn is_retryable(err: &FetchError) -> bool {
matches!(err, FetchError::Http(_) | FetchError::Transient(_))
}
/// misskey.io media hosts need no extra headers (verified: direct GET works).
pub fn media_headers(_url: &str) -> Option<Vec<(&'static str, String)>> {
None
}
/// Fetches a note from misskey.io by id. The API answers client failures
/// with HTTP 400 + `{"error":{"code":...}}` (NO_SUCH_NOTE → NotFound);
/// everything else non-success is transient and retried by [`crate::site::fetch`].
+1 -3
View File
@@ -1,6 +1,4 @@
mod interface;
mod model;
pub use interface::{
MisskeySite, PATTERN, cache_key, enabled, fetch_from_url, is_retryable, media_headers,
};
pub use interface::{MisskeySite, PATTERN, cache_key, fetch_from_url};
+4 -2
View File
@@ -891,8 +891,10 @@ mod tests {
};
assert_eq!(err.to_string(), "example error: boom");
assert!(err.source().is_some());
// Permanent by default: no site's is_retryable matches it.
assert!(!twitter::is_retryable(&err));
// Permanent by default: no site's is_retryable matches it (the trait
// default is the policy for every site that does not override it).
assert!(!twitter::TwitterSite.is_retryable(&err));
assert!(twitter::TwitterSite.is_retryable(&FetchError::Transient("429".into())));
}
#[test]
+6 -36
View File
@@ -30,8 +30,12 @@ pub static PATTERN: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(r"^(?:https?://)?(?:www\.|mobile\.)?(?:x|twitter|fixvx|vxtwitter|fixupx|fxtwitter)\.com/[^.]+/status/(\d+)").unwrap()
});
pub fn enabled() -> bool {
true
/// Cache key for a twitter URL: `"twitter:<id>"`. The prefix is the site id
/// used for caption-format lookup and link-cache keys.
pub fn cache_key(url: &str) -> Option<String> {
PATTERN
.captures(url)
.map(|caps| format!("twitter:{}", &caps[1]))
}
pub async fn fetch_from_url(url: &str) -> Result<Fetched, FetchError> {
@@ -68,26 +72,6 @@ pub async fn fetch_from_url(url: &str) -> Result<Fetched, FetchError> {
}
}
/// Cache key for a twitter URL: `"twitter:<id>"`. The prefix is the site id
/// used for caption-format lookup and link-cache keys.
pub fn cache_key(url: &str) -> Option<String> {
PATTERN
.captures(url)
.map(|caps| format!("twitter:{}", &caps[1]))
}
/// Twitter's fetch-retry policy: transient classes only. Not-found, blocked,
/// sensitive (NSFW withholding) and parse failures are permanent — retrying
/// them only wastes attempts against the syndication endpoint.
pub fn is_retryable(err: &FetchError) -> bool {
matches!(err, FetchError::Http(_) | FetchError::Transient(_))
}
/// twimg URLs need no extra headers (no hotlink protection).
pub fn media_headers(_url: &str) -> Option<Vec<(&'static str, String)>> {
None
}
/// Fetches a tweet from the syndication endpoint. Deleted/blocked tweets
/// surface as `FetchError::NotFound`; withheld content (empty tombstone,
/// age-restricted) as `FetchError::Sensitive`.
@@ -462,20 +446,6 @@ mod tests {
assert_eq!(cache_key("https://example.com/1"), None);
}
#[test]
fn is_retryable_classifies_transient_and_permanent() {
// Transient: network errors and explicit transient statuses (the
// `Http` arm shares this match arm with `Transient`).
assert!(is_retryable(&FetchError::Transient("429".into())));
// Permanent: gone, blocked, withheld, oversized, unparseable.
assert!(!is_retryable(&FetchError::NotFound));
assert!(!is_retryable(&FetchError::Blocked));
assert!(!is_retryable(&FetchError::Sensitive));
assert!(!is_retryable(&FetchError::TooLarge));
let json_err = serde_json::from_str::<serde_json::Value>("x").unwrap_err();
assert!(!is_retryable(&FetchError::Json(json_err)));
}
#[test]
fn syndication_json_converts_to_fetched() {
let raw = fixture(serde_json::json!([
+1 -3
View File
@@ -2,6 +2,4 @@ mod auth;
mod interface;
mod model;
pub use interface::{
PATTERN, Tweet, TwitterSite, cache_key, enabled, fetch_from_url, is_retryable, media_headers,
};
pub use interface::{PATTERN, Tweet, TwitterSite, cache_key, fetch_from_url};