mirror of
https://github.com/TheFunny/TelegramTwitterMediaBot.git
synced 2026-09-23 23:32:05 +00:00
handlers: dedup extracted URLs by normalized post id
Exact-string dedup let https://x.com/u/status/1 and https://x.com/u/status/1/photo/1 (or the same link in text and caption) through twice, causing duplicate fetches and sends. Dedup now uses cache_key, falling back to the raw URL for unsupported links.
This commit is contained in:
@@ -136,7 +136,10 @@ pub fn extract_urls(message: &Message) -> Vec<String> {
|
||||
}
|
||||
}
|
||||
let mut seen = HashSet::new();
|
||||
urls.retain(|url| seen.insert(url.clone()));
|
||||
// Dedup by the normalized post id so variant URLs of the same post
|
||||
// (/status/1 vs /status/1/photo/1) are sent once; unsupported URLs fall
|
||||
// back to exact-string dedup.
|
||||
urls.retain(|url| seen.insert(x_media::site::cache_key(url).unwrap_or_else(|| url.clone())));
|
||||
urls
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user