mirror of
https://github.com/TheFunny/TelegramTwitterMediaBot.git
synced 2026-09-23 23:32:05 +00:00
perf: stop probing a media item's size before downloading it
The upload fallback asked `x_media::site::media_size` for every remote item
before downloading it. That call is a real GET (not a HEAD) on the *un-
guarded* `CLIENT` — so every fallback item cost two requests where one would
do, the response body was never consumed (the connection cannot return to
the pool), and for photos the answer was discarded outright
(`too_large && !matches!(item, Photo { .. })` still fired the request). It
bypassed `media_request`'s private-network guard as well, the one choke
point every other egress goes through.
For videos the probe was redundant twice over: `download_media_limited`
reads the declared Content-Length before any body byte and aborts with
`FetchError::TooLarge`, which the call site already turns into the item's
smaller URL (`FallbackError::MediaTooLarge` → `fallback_url`).
`media_size` is deleted (no other caller) and the download's own cap is the
only size gate. The video cap is now exactly `MAX_UPLOAD_BYTES` instead of
`MAX_UPLOAD_BYTES + 1`, so the boundary the probe drew survives byte for
byte: a file of exactly the cap is admitted (`len > max_bytes` is false),
one byte over degrades to the smaller URL.
Verified with a throwaway harness (a local HTTP server reached through
`TELOXIDE_PROXY`, the one LAN egress the guard allows): a small video, a
photo and an oversized video each cost 1 request where the probe made it 2,
and the oversized one still lands on `/fallback.mp4` without fetching it.
`cargo fmt --check`, `cargo clippy --workspace --all-targets --locked -- -D
warnings` and `cargo test --workspace --locked` clean.
This commit is contained in:
@@ -656,21 +656,6 @@ fn apply_media_headers(mut request: reqwest::RequestBuilder, url: &str) -> reqwe
|
||||
request
|
||||
}
|
||||
|
||||
/// Downloads media bytes for the bot's upload fallback: when Telegram's own
|
||||
/// fetch of a media URL is blocked (hotlink protection), the bot downloads
|
||||
/// the file itself and uploads it via multipart. Site-appropriate headers
|
||||
/// come from each site's `media_headers` (pixiv image hosts need `Referer`).
|
||||
/// Returns the Content-Length of a media URL, or `None` when the server does
|
||||
/// not report one. Used to check whether a file fits Telegram's size limits
|
||||
/// before downloading/uploading it.
|
||||
pub async fn media_size(url: &str) -> Result<Option<u64>, FetchError> {
|
||||
let response = apply_media_headers(CLIENT.get(url), url)
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?;
|
||||
Ok(response.content_length())
|
||||
}
|
||||
|
||||
/// Maps a media download's HTTP status onto the same classes the site
|
||||
/// adapters use, so callers can tell "try again" from "this URL is dead":
|
||||
/// 4xx is a property of the media (gone, refused by the host), while 429/5xx
|
||||
@@ -687,7 +672,13 @@ fn download_status_error(status: reqwest::StatusCode) -> FetchError {
|
||||
/// Downloads a media file with a hard size cap: the body is streamed and the
|
||||
/// download aborts with [`FetchError::TooLarge`] the moment the cap is
|
||||
/// crossed (or when a declared Content-Length already exceeds it). Keeps the
|
||||
/// bot from buffering arbitrarily large bodies into memory.
|
||||
/// bot from buffering arbitrarily large bodies into memory — the size check
|
||||
/// the bot's upload fallback needs is the one here, not a probe of its own.
|
||||
///
|
||||
/// This is the bot's download path for the upload fallback: when Telegram
|
||||
/// cannot fetch a media URL itself (hotlink protection), the bot downloads
|
||||
/// the file and uploads it via multipart. Site-appropriate headers come from
|
||||
/// each site's `media_headers` (pixiv image hosts need `Referer`).
|
||||
pub async fn download_media_limited(url: &str, max_bytes: u64) -> Result<bytes::Bytes, FetchError> {
|
||||
let response = send_download(media_request(url)?).await?;
|
||||
if let Some(len) = response.content_length()
|
||||
|
||||
@@ -62,12 +62,14 @@ async fn download_to_temp(
|
||||
| MediaItemPayload::Animation { media, .. } => media,
|
||||
};
|
||||
// Photos are downloaded even over the upload cap so `prepare_photo` can
|
||||
// downscale / transcode them (cap = decode budget); videos/animations
|
||||
// abort as soon as the upload cap is crossed mid-stream.
|
||||
// downscale / transcode them (cap = decode budget); videos and animations
|
||||
// are refused as soon as the declared size crosses the upload cap — the
|
||||
// boundary the size probe this replaced drew: a file of exactly the cap is
|
||||
// admitted (`len > max_bytes` is false), one byte over is not.
|
||||
let limit = if matches!(item, MediaItemPayload::Photo { .. }) {
|
||||
photo::MAX_DECODE_BYTES
|
||||
} else {
|
||||
MAX_UPLOAD_BYTES + 1
|
||||
MAX_UPLOAD_BYTES
|
||||
};
|
||||
let bytes = match x_media::site::download_media_limited(media_url, limit).await {
|
||||
Ok(bytes) => bytes,
|
||||
@@ -197,28 +199,13 @@ pub(super) async fn prepare_upload_item(
|
||||
keep_alive: None,
|
||||
});
|
||||
}
|
||||
// Size check before downloading/uploading: over the cap, use the
|
||||
// smaller URL instead of the file. Photos are exempt — they are
|
||||
// downloaded and processed (downscale / PNG→JPEG) before uploading.
|
||||
let too_large = match x_media::site::media_size(media_url).await {
|
||||
Ok(Some(size)) => size > MAX_UPLOAD_BYTES,
|
||||
_ => false,
|
||||
};
|
||||
let too_large = too_large && !matches!(item, MediaItemPayload::Photo { .. });
|
||||
if too_large {
|
||||
let url = item
|
||||
.fallback_url()
|
||||
.ok_or_else(|| FallbackError::Permanent {
|
||||
message: "media too large".into(),
|
||||
})?;
|
||||
let media = media_from_url(&item, url, caption, item.thumbnail_url())
|
||||
.map_err(|message| FallbackError::Permanent { message })?;
|
||||
return Ok(PreparedItem {
|
||||
index,
|
||||
media,
|
||||
keep_alive: None,
|
||||
});
|
||||
}
|
||||
// Whether a file is over the cap is settled by the download itself:
|
||||
// `download_media_limited` reads the declared Content-Length before any
|
||||
// body byte and aborts with `FetchError::TooLarge`, which arrives here as
|
||||
// `FallbackError::MediaTooLarge` — turned into the item's smaller URL by
|
||||
// the match below. A separate size probe used to issue a second GET of the
|
||||
// same URL for an answer this path already has (and issued it for photos,
|
||||
// whose answer was discarded one line later).
|
||||
match download_to_temp(&item).await {
|
||||
Ok((file, bytes)) => {
|
||||
if matches!(item, MediaItemPayload::Photo { .. }) {
|
||||
|
||||
Reference in New Issue
Block a user