mirror of
https://github.com/TheFunny/TelegramTwitterMediaBot.git
synced 2026-09-23 23:32:05 +00:00
perf(photo): stop re-reading the downloaded temp file
download_to_temp buffered the full bytes, wrote them to a temp file, and prepare_photo then read the whole file back from disk. The bytes are already in memory — pass them through (download_to_temp now returns (file, bytes)) so photo processing never touches the disk for input. Adds the bytes dependency to xmedia-bot (already in the lock via x-media).
This commit is contained in:
Generated
+1
@@ -2946,6 +2946,7 @@ dependencies = [
|
||||
name = "xmedia-bot"
|
||||
version = "1.1.1"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"dotenv",
|
||||
"fast_image_resize",
|
||||
"html-escape",
|
||||
|
||||
@@ -17,6 +17,7 @@ rusqlite = { version = "0.32", features = ["bundled"] }
|
||||
rand = "0.8"
|
||||
tempfile = "3"
|
||||
parking_lot = "0.12"
|
||||
bytes = "1"
|
||||
png = "0.18"
|
||||
zune-jpeg = "0.5"
|
||||
fast_image_resize = "6"
|
||||
|
||||
@@ -67,8 +67,9 @@ impl PixBuf {
|
||||
}
|
||||
|
||||
/// Entry point: detects the format and processes the photo if needed.
|
||||
pub fn prepare_photo(file: NamedTempFile) -> Result<PhotoPrep, String> {
|
||||
let bytes = std::fs::read(file.path()).map_err(|e| format!("prepare read failed: {e}"))?;
|
||||
/// The caller hands in the already-downloaded bytes (they are in memory from
|
||||
/// the download anyway; re-reading the temp file would double the I/O).
|
||||
pub fn prepare_photo(file: NamedTempFile, bytes: &[u8]) -> Result<PhotoPrep, String> {
|
||||
if bytes.starts_with(b"\x89PNG\r\n\x1a\n") {
|
||||
prepare_png(file, bytes)
|
||||
} else if bytes.starts_with(&[0xFF, 0xD8, 0xFF]) {
|
||||
@@ -216,8 +217,8 @@ fn target_dims(w: u32, h: u32) -> (u32, u32) {
|
||||
/// PNG branch: decode (16→8, palette→RGB; gray/GA stay), flatten RGBA to
|
||||
/// RGB, Lanczos-downscale beyond the dimension cap, encode PNG — a PNG still
|
||||
/// over the upload cap afterwards becomes JPEG.
|
||||
fn prepare_png(file: NamedTempFile, bytes: Vec<u8>) -> Result<PhotoPrep, String> {
|
||||
let (w, h, _bit_depth, color_type) = parse_png_header(&bytes).ok_or("invalid PNG header")?;
|
||||
fn prepare_png(file: NamedTempFile, bytes: &[u8]) -> Result<PhotoPrep, String> {
|
||||
let (w, h, _bit_depth, color_type) = parse_png_header(bytes).ok_or("invalid PNG header")?;
|
||||
let size_over = bytes.len() as u64 > MAX_UPLOAD_BYTES;
|
||||
if w + h <= PHOTO_MAX_DIMENSION_SUM && !size_over {
|
||||
return Ok(PhotoPrep::Upload(file));
|
||||
@@ -239,7 +240,7 @@ fn prepare_png(file: NamedTempFile, bytes: Vec<u8>) -> Result<PhotoPrep, String>
|
||||
png::ColorType::Indexed => png::Transformations::EXPAND,
|
||||
_ => png::Transformations::STRIP_16,
|
||||
};
|
||||
let mut decoder = png::Decoder::new(std::io::Cursor::new(&bytes));
|
||||
let mut decoder = png::Decoder::new(std::io::Cursor::new(bytes));
|
||||
decoder.set_transformations(transforms);
|
||||
let mut reader = decoder
|
||||
.read_info()
|
||||
@@ -286,8 +287,8 @@ fn prepare_png(file: NamedTempFile, bytes: Vec<u8>) -> Result<PhotoPrep, String>
|
||||
}
|
||||
|
||||
/// JPEG branch: zune-jpeg decode → Lanczos downscale → jpeg-encoder output.
|
||||
fn prepare_jpeg(file: NamedTempFile, bytes: Vec<u8>) -> Result<PhotoPrep, String> {
|
||||
let mut decoder = zune_jpeg::JpegDecoder::new(std::io::Cursor::new(&bytes));
|
||||
fn prepare_jpeg(file: NamedTempFile, bytes: &[u8]) -> Result<PhotoPrep, String> {
|
||||
let mut decoder = zune_jpeg::JpegDecoder::new(std::io::Cursor::new(bytes));
|
||||
// Decodes to RGB by default. Headers first so dimensions are known before
|
||||
// the (potentially huge) pixel decode.
|
||||
decoder
|
||||
@@ -423,7 +424,7 @@ mod tests {
|
||||
|
||||
let mut file = tempfile::Builder::new().suffix(".png").tempfile().unwrap();
|
||||
std::io::Write::write_all(file.as_file_mut(), &bytes).unwrap();
|
||||
prepare_photo(file)
|
||||
prepare_photo(file, &bytes)
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -468,7 +469,7 @@ mod tests {
|
||||
}
|
||||
let mut file = tempfile::Builder::new().suffix(".jpg").tempfile().unwrap();
|
||||
std::io::Write::write_all(file.as_file_mut(), &bytes).unwrap();
|
||||
match prepare_photo(file).unwrap() {
|
||||
match prepare_photo(file, &bytes).unwrap() {
|
||||
PhotoPrep::Upload(file) => {
|
||||
let out = std::fs::read(file.path()).unwrap();
|
||||
assert!(out.starts_with(&[0xFF, 0xD8]), "output must stay jpeg");
|
||||
@@ -516,7 +517,7 @@ mod tests {
|
||||
|
||||
let mut file = tempfile::Builder::new().suffix(".png").tempfile().unwrap();
|
||||
std::io::Write::write_all(file.as_file_mut(), &bytes).unwrap();
|
||||
match prepare_photo(file).unwrap() {
|
||||
match prepare_photo(file, &bytes).unwrap() {
|
||||
PhotoPrep::Upload(file) => {
|
||||
let out = std::fs::read(file.path()).unwrap();
|
||||
assert!(out.starts_with(&[0xFF, 0xD8]), "must transcode to JPEG");
|
||||
|
||||
@@ -556,9 +556,14 @@ enum FallbackError {
|
||||
/// only if still too big. Anything that cannot be fixed falls back to the
|
||||
/// item's smaller URL.
|
||||
///
|
||||
/// Downloads one media item to a temp file (deleted on drop). Network errors
|
||||
/// are retryable; size over the upload cap and other download errors are not.
|
||||
async fn download_to_temp(item: &MediaItemPayload) -> Result<NamedTempFile, FallbackError> {
|
||||
/// Downloads one media item to a temp file (deleted on drop), returning the
|
||||
/// file plus the downloaded bytes (photos keep the bytes for
|
||||
/// [`photo::prepare_photo`] — re-reading the file would double the I/O).
|
||||
/// Network errors are retryable; size over the upload cap and other download
|
||||
/// errors are not.
|
||||
async fn download_to_temp(
|
||||
item: &MediaItemPayload,
|
||||
) -> Result<(NamedTempFile, bytes::Bytes), FallbackError> {
|
||||
let media_url = match item {
|
||||
MediaItemPayload::Photo { media, .. }
|
||||
| MediaItemPayload::Video { media, .. }
|
||||
@@ -601,7 +606,7 @@ async fn download_to_temp(item: &MediaItemPayload) -> Result<NamedTempFile, Fall
|
||||
.map_err(|e| FallbackError::Permanent {
|
||||
message: format!("temp file write failed: {e}"),
|
||||
})?;
|
||||
Ok(file)
|
||||
Ok((file, bytes))
|
||||
}
|
||||
|
||||
/// Builds the media group item from an uploaded file.
|
||||
@@ -711,14 +716,14 @@ async fn prepare_upload_item(
|
||||
});
|
||||
}
|
||||
match download_to_temp(&item).await {
|
||||
Ok(file) => {
|
||||
Ok((file, bytes)) => {
|
||||
if matches!(item, MediaItemPayload::Photo { .. }) {
|
||||
// Telegram rejects photos wider+taller than 10000 px combined
|
||||
// (PHOTO_INVALID_DIMENSIONS): downscale the downloaded file
|
||||
// before uploading; photos that cannot be brought within the
|
||||
// limits degrade to the smaller URL. CPU-heavy work runs off
|
||||
// the async executor thread.
|
||||
let prep = tokio::task::spawn_blocking(move || photo::prepare_photo(file))
|
||||
let prep = tokio::task::spawn_blocking(move || photo::prepare_photo(file, &bytes))
|
||||
.await
|
||||
.map_err(|e| FallbackError::Permanent {
|
||||
message: format!("photo worker panicked: {e}"),
|
||||
@@ -1047,7 +1052,7 @@ pub async fn send_animation(bot: &Bot, task: &Task) -> Result<Vec<i64>, SendErro
|
||||
media_url
|
||||
);
|
||||
match download_to_temp(animation).await {
|
||||
Ok(file) => {
|
||||
Ok((file, _bytes)) => {
|
||||
let path = file.path().to_path_buf();
|
||||
match send_animation_inner(
|
||||
bot,
|
||||
|
||||
Reference in New Issue
Block a user