refactor(x-media): drop the per-site fns the Site defaults already cover

twitter, bsky, misskey and bilibili each carried enabled() -> true,
media_headers(url) -> None and is_retryable(err) -> the trait's own default,
with no caller outside their tests (the adapters never override those
methods, so the default was already the production policy). The tests that
only restated the default are gone; the two that pin site-specific classes
(bsky's MediaPrep, bilibili's risk-control codes) now ask the Site impl, and
site/mod.rs keeps one assertion of the shared retry policy. The live
verification notes (no Referer needed for hdslb/twimg) survive as comments.
This commit is contained in:
2026-09-21 17:12:42 +08:00
parent 5630a86d88
commit 9a7cda05c9
9 changed files with 22 additions and 100 deletions
+5 -18
View File
@@ -111,10 +111,6 @@ pub static PATTERN: LazyLock<Regex> = LazyLock::new(|| {
.unwrap() .unwrap()
}); });
pub fn enabled() -> bool {
true
}
pub async fn fetch_from_url(url: &str) -> Result<Fetched, FetchError> { pub async fn fetch_from_url(url: &str) -> Result<Fetched, FetchError> {
let dynamic_id = PATTERN let dynamic_id = PATTERN
.captures(url) .captures(url)
@@ -133,18 +129,9 @@ pub fn cache_key(url: &str) -> Option<String> {
.map(|caps| format!("bilibili:{}", &caps[1])) .map(|caps| format!("bilibili:{}", &caps[1]))
} }
/// Bilibili's fetch-retry policy: transient classes only. Not-found, blocked // hdslb media serves without a `Referer` (verified live 2026-09-17 on
/// and parse failures are permanent. // `i0.hdslb.com` image URLs, requested both with and without one), so this
pub fn is_retryable(err: &FetchError) -> bool { // adapter does not override `Site::media_headers`.
matches!(err, FetchError::Http(_) | FetchError::Transient(_))
}
/// hdslb media serves without a `Referer` (verified live 2026-09-17 on
/// `i0.hdslb.com` image URLs, requested both with and without one), so no
/// extra headers.
pub fn media_headers(_url: &str) -> Option<Vec<(&'static str, String)>> {
None
}
/// `Cookie` header for bilibili requests: the operator's `BILIBILI_COOKIE` /// `Cookie` header for bilibili requests: the operator's `BILIBILI_COOKIE`
/// when set, otherwise the anonymous device cookies. /// when set, otherwise the anonymous device cookies.
@@ -908,7 +895,7 @@ mod tests {
// dropping the post. // dropping the post.
for code in [-352, -412] { for code in [-352, -412] {
let err = code_error(code, "-352").unwrap(); let err = code_error(code, "-352").unwrap();
assert!(is_retryable(&err), "{err}"); assert!(BilibiliSite.is_retryable(&err), "{err}");
} }
// A removed dynamic is permanent. // A removed dynamic is permanent.
assert!(matches!(code_error(500, ""), Some(FetchError::NotFound))); assert!(matches!(code_error(500, ""), Some(FetchError::NotFound)));
@@ -917,7 +904,7 @@ mod tests {
Some(FetchError::NotFound) Some(FetchError::NotFound)
)); ));
let err = code_error(-400, "param parsing failed").unwrap(); let err = code_error(-400, "param parsing failed").unwrap();
assert!(!is_retryable(&err), "{err}"); assert!(!BilibiliSite.is_retryable(&err), "{err}");
assert!(err.to_string().contains("-400"), "{err}"); assert!(err.to_string().contains("-400"), "{err}");
} }
+1 -3
View File
@@ -1,6 +1,4 @@
mod interface; mod interface;
mod model; mod model;
pub use interface::{ pub use interface::{BilibiliSite, PATTERN, cache_key, fetch_from_url};
BilibiliSite, PATTERN, cache_key, enabled, fetch_from_url, is_retryable, media_headers,
};
+3 -17
View File
@@ -30,10 +30,6 @@ pub static PATTERN: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(r"^(?:https?://)?bsky\.app/profile/([\w.\-:]+)/post/([\w.\-~]+)").unwrap() Regex::new(r"^(?:https?://)?bsky\.app/profile/([\w.\-:]+)/post/([\w.\-~]+)").unwrap()
}); });
pub fn enabled() -> bool {
true
}
pub async fn fetch_from_url(url: &str) -> Result<Fetched, FetchError> { pub async fn fetch_from_url(url: &str) -> Result<Fetched, FetchError> {
let caps = PATTERN.captures(url).ok_or(FetchError::NotFound)?; let caps = PATTERN.captures(url).ok_or(FetchError::NotFound)?;
let handle = caps let handle = caps
@@ -112,17 +108,6 @@ pub fn cache_key(url: &str) -> Option<String> {
.map(|caps| format!("bsky:{}/{}", &caps[1], &caps[2])) .map(|caps| format!("bsky:{}/{}", &caps[1], &caps[2]))
} }
/// Bluesky's fetch-retry policy: transient classes only. Not-found, blocked
/// and parse failures are permanent.
pub fn is_retryable(err: &FetchError) -> bool {
matches!(err, FetchError::Http(_) | FetchError::Transient(_))
}
/// bsky media (cdn.bsky.app) needs no extra headers.
pub fn media_headers(_url: &str) -> Option<Vec<(&'static str, String)>> {
None
}
/// Segments fetched (and written) at once while remuxing an HLS video. Small /// Segments fetched (and written) at once while remuxing an HLS video. Small
/// on purpose: a segment can be up to 20 MiB and the whole playlist is capped /// on purpose: a segment can be up to 20 MiB and the whole playlist is capped
/// at 256 MiB, so this is also what bounds the remux's peak memory. /// at 256 MiB, so this is also what bounds the remux's peak memory.
@@ -499,10 +484,11 @@ mod tests {
/// ([`fetch_hls`]). The classes below are the ones still retried there. /// ([`fetch_hls`]). The classes below are the ones still retried there.
#[test] #[test]
fn media_prep_failure_is_not_retried() { fn media_prep_failure_is_not_retried() {
assert!(!is_retryable(&FetchError::MediaPrep( use crate::site::Site as _;
assert!(!BskySite.is_retryable(&FetchError::MediaPrep(
"bsky video remux failed: segment 400: 503".into() "bsky video remux failed: segment 400: 503".into()
))); )));
assert!(is_retryable(&FetchError::Transient("429".into()))); assert!(BskySite.is_retryable(&FetchError::Transient("429".into())));
} }
#[test] #[test]
+1 -3
View File
@@ -1,6 +1,4 @@
mod interface; mod interface;
mod model; mod model;
pub use interface::{ pub use interface::{BskySite, PATTERN, Post, cache_key, fetch_from_url};
BskySite, PATTERN, Post, cache_key, enabled, fetch_from_url, is_retryable, media_headers,
};
@@ -34,10 +34,6 @@ impl Site for MisskeySite {
pub static PATTERN: LazyLock<Regex> = pub static PATTERN: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"^(?:https?://)?misskey\.io/notes/([\w.\-~]+)").unwrap()); LazyLock::new(|| Regex::new(r"^(?:https?://)?misskey\.io/notes/([\w.\-~]+)").unwrap());
pub fn enabled() -> bool {
true
}
pub async fn fetch_from_url(url: &str) -> Result<Fetched, FetchError> { pub async fn fetch_from_url(url: &str) -> Result<Fetched, FetchError> {
let caps = PATTERN.captures(url).ok_or(FetchError::NotFound)?; let caps = PATTERN.captures(url).ok_or(FetchError::NotFound)?;
let note_id = caps.get(1).ok_or(FetchError::NotFound)?.as_str(); let note_id = caps.get(1).ok_or(FetchError::NotFound)?.as_str();
@@ -53,17 +49,6 @@ pub fn cache_key(url: &str) -> Option<String> {
.map(|caps| format!("misskey:{}", &caps[1])) .map(|caps| format!("misskey:{}", &caps[1]))
} }
/// Misskey's fetch-retry policy: transient classes only. Not-found, blocked
/// and parse failures are permanent.
pub fn is_retryable(err: &FetchError) -> bool {
matches!(err, FetchError::Http(_) | FetchError::Transient(_))
}
/// misskey.io media hosts need no extra headers (verified: direct GET works).
pub fn media_headers(_url: &str) -> Option<Vec<(&'static str, String)>> {
None
}
/// Fetches a note from misskey.io by id. The API answers client failures /// Fetches a note from misskey.io by id. The API answers client failures
/// with HTTP 400 + `{"error":{"code":...}}` (NO_SUCH_NOTE → NotFound); /// with HTTP 400 + `{"error":{"code":...}}` (NO_SUCH_NOTE → NotFound);
/// everything else non-success is transient and retried by [`crate::site::fetch`]. /// everything else non-success is transient and retried by [`crate::site::fetch`].
+1 -3
View File
@@ -1,6 +1,4 @@
mod interface; mod interface;
mod model; mod model;
pub use interface::{ pub use interface::{MisskeySite, PATTERN, cache_key, fetch_from_url};
MisskeySite, PATTERN, cache_key, enabled, fetch_from_url, is_retryable, media_headers,
};
+4 -2
View File
@@ -891,8 +891,10 @@ mod tests {
}; };
assert_eq!(err.to_string(), "example error: boom"); assert_eq!(err.to_string(), "example error: boom");
assert!(err.source().is_some()); assert!(err.source().is_some());
// Permanent by default: no site's is_retryable matches it. // Permanent by default: no site's is_retryable matches it (the trait
assert!(!twitter::is_retryable(&err)); // default is the policy for every site that does not override it).
assert!(!twitter::TwitterSite.is_retryable(&err));
assert!(twitter::TwitterSite.is_retryable(&FetchError::Transient("429".into())));
} }
#[test] #[test]
+6 -36
View File
@@ -30,8 +30,12 @@ pub static PATTERN: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(r"^(?:https?://)?(?:www\.|mobile\.)?(?:x|twitter|fixvx|vxtwitter|fixupx|fxtwitter)\.com/[^.]+/status/(\d+)").unwrap() Regex::new(r"^(?:https?://)?(?:www\.|mobile\.)?(?:x|twitter|fixvx|vxtwitter|fixupx|fxtwitter)\.com/[^.]+/status/(\d+)").unwrap()
}); });
pub fn enabled() -> bool { /// Cache key for a twitter URL: `"twitter:<id>"`. The prefix is the site id
true /// used for caption-format lookup and link-cache keys.
pub fn cache_key(url: &str) -> Option<String> {
PATTERN
.captures(url)
.map(|caps| format!("twitter:{}", &caps[1]))
} }
pub async fn fetch_from_url(url: &str) -> Result<Fetched, FetchError> { pub async fn fetch_from_url(url: &str) -> Result<Fetched, FetchError> {
@@ -68,26 +72,6 @@ pub async fn fetch_from_url(url: &str) -> Result<Fetched, FetchError> {
} }
} }
/// Cache key for a twitter URL: `"twitter:<id>"`. The prefix is the site id
/// used for caption-format lookup and link-cache keys.
pub fn cache_key(url: &str) -> Option<String> {
PATTERN
.captures(url)
.map(|caps| format!("twitter:{}", &caps[1]))
}
/// Twitter's fetch-retry policy: transient classes only. Not-found, blocked,
/// sensitive (NSFW withholding) and parse failures are permanent — retrying
/// them only wastes attempts against the syndication endpoint.
pub fn is_retryable(err: &FetchError) -> bool {
matches!(err, FetchError::Http(_) | FetchError::Transient(_))
}
/// twimg URLs need no extra headers (no hotlink protection).
pub fn media_headers(_url: &str) -> Option<Vec<(&'static str, String)>> {
None
}
/// Fetches a tweet from the syndication endpoint. Deleted/blocked tweets /// Fetches a tweet from the syndication endpoint. Deleted/blocked tweets
/// surface as `FetchError::NotFound`; withheld content (empty tombstone, /// surface as `FetchError::NotFound`; withheld content (empty tombstone,
/// age-restricted) as `FetchError::Sensitive`. /// age-restricted) as `FetchError::Sensitive`.
@@ -462,20 +446,6 @@ mod tests {
assert_eq!(cache_key("https://example.com/1"), None); assert_eq!(cache_key("https://example.com/1"), None);
} }
#[test]
fn is_retryable_classifies_transient_and_permanent() {
// Transient: network errors and explicit transient statuses (the
// `Http` arm shares this match arm with `Transient`).
assert!(is_retryable(&FetchError::Transient("429".into())));
// Permanent: gone, blocked, withheld, oversized, unparseable.
assert!(!is_retryable(&FetchError::NotFound));
assert!(!is_retryable(&FetchError::Blocked));
assert!(!is_retryable(&FetchError::Sensitive));
assert!(!is_retryable(&FetchError::TooLarge));
let json_err = serde_json::from_str::<serde_json::Value>("x").unwrap_err();
assert!(!is_retryable(&FetchError::Json(json_err)));
}
#[test] #[test]
fn syndication_json_converts_to_fetched() { fn syndication_json_converts_to_fetched() {
let raw = fixture(serde_json::json!([ let raw = fixture(serde_json::json!([
+1 -3
View File
@@ -2,6 +2,4 @@ mod auth;
mod interface; mod interface;
mod model; mod model;
pub use interface::{ pub use interface::{PATTERN, Tweet, TwitterSite, cache_key, fetch_from_url};
PATTERN, Tweet, TwitterSite, cache_key, enabled, fetch_from_url, is_retryable, media_headers,
};