mirror of
https://github.com/TheFunny/TelegramTwitterMediaBot.git
synced 2026-09-23 23:32:05 +00:00
perf(photo): stop re-reading the downloaded temp file
download_to_temp buffered the full bytes, wrote them to a temp file, and prepare_photo then read the whole file back from disk. The bytes are already in memory — pass them through (download_to_temp now returns (file, bytes)) so photo processing never touches the disk for input. Adds the bytes dependency to xmedia-bot (already in the lock via x-media).
This commit is contained in:
Generated
+1
@@ -2946,6 +2946,7 @@ dependencies = [
|
|||||||
name = "xmedia-bot"
|
name = "xmedia-bot"
|
||||||
version = "1.1.1"
|
version = "1.1.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"bytes",
|
||||||
"dotenv",
|
"dotenv",
|
||||||
"fast_image_resize",
|
"fast_image_resize",
|
||||||
"html-escape",
|
"html-escape",
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ rusqlite = { version = "0.32", features = ["bundled"] }
|
|||||||
rand = "0.8"
|
rand = "0.8"
|
||||||
tempfile = "3"
|
tempfile = "3"
|
||||||
parking_lot = "0.12"
|
parking_lot = "0.12"
|
||||||
|
bytes = "1"
|
||||||
png = "0.18"
|
png = "0.18"
|
||||||
zune-jpeg = "0.5"
|
zune-jpeg = "0.5"
|
||||||
fast_image_resize = "6"
|
fast_image_resize = "6"
|
||||||
|
|||||||
@@ -67,8 +67,9 @@ impl PixBuf {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Entry point: detects the format and processes the photo if needed.
|
/// Entry point: detects the format and processes the photo if needed.
|
||||||
pub fn prepare_photo(file: NamedTempFile) -> Result<PhotoPrep, String> {
|
/// The caller hands in the already-downloaded bytes (they are in memory from
|
||||||
let bytes = std::fs::read(file.path()).map_err(|e| format!("prepare read failed: {e}"))?;
|
/// the download anyway; re-reading the temp file would double the I/O).
|
||||||
|
pub fn prepare_photo(file: NamedTempFile, bytes: &[u8]) -> Result<PhotoPrep, String> {
|
||||||
if bytes.starts_with(b"\x89PNG\r\n\x1a\n") {
|
if bytes.starts_with(b"\x89PNG\r\n\x1a\n") {
|
||||||
prepare_png(file, bytes)
|
prepare_png(file, bytes)
|
||||||
} else if bytes.starts_with(&[0xFF, 0xD8, 0xFF]) {
|
} else if bytes.starts_with(&[0xFF, 0xD8, 0xFF]) {
|
||||||
@@ -216,8 +217,8 @@ fn target_dims(w: u32, h: u32) -> (u32, u32) {
|
|||||||
/// PNG branch: decode (16→8, palette→RGB; gray/GA stay), flatten RGBA to
|
/// PNG branch: decode (16→8, palette→RGB; gray/GA stay), flatten RGBA to
|
||||||
/// RGB, Lanczos-downscale beyond the dimension cap, encode PNG — a PNG still
|
/// RGB, Lanczos-downscale beyond the dimension cap, encode PNG — a PNG still
|
||||||
/// over the upload cap afterwards becomes JPEG.
|
/// over the upload cap afterwards becomes JPEG.
|
||||||
fn prepare_png(file: NamedTempFile, bytes: Vec<u8>) -> Result<PhotoPrep, String> {
|
fn prepare_png(file: NamedTempFile, bytes: &[u8]) -> Result<PhotoPrep, String> {
|
||||||
let (w, h, _bit_depth, color_type) = parse_png_header(&bytes).ok_or("invalid PNG header")?;
|
let (w, h, _bit_depth, color_type) = parse_png_header(bytes).ok_or("invalid PNG header")?;
|
||||||
let size_over = bytes.len() as u64 > MAX_UPLOAD_BYTES;
|
let size_over = bytes.len() as u64 > MAX_UPLOAD_BYTES;
|
||||||
if w + h <= PHOTO_MAX_DIMENSION_SUM && !size_over {
|
if w + h <= PHOTO_MAX_DIMENSION_SUM && !size_over {
|
||||||
return Ok(PhotoPrep::Upload(file));
|
return Ok(PhotoPrep::Upload(file));
|
||||||
@@ -239,7 +240,7 @@ fn prepare_png(file: NamedTempFile, bytes: Vec<u8>) -> Result<PhotoPrep, String>
|
|||||||
png::ColorType::Indexed => png::Transformations::EXPAND,
|
png::ColorType::Indexed => png::Transformations::EXPAND,
|
||||||
_ => png::Transformations::STRIP_16,
|
_ => png::Transformations::STRIP_16,
|
||||||
};
|
};
|
||||||
let mut decoder = png::Decoder::new(std::io::Cursor::new(&bytes));
|
let mut decoder = png::Decoder::new(std::io::Cursor::new(bytes));
|
||||||
decoder.set_transformations(transforms);
|
decoder.set_transformations(transforms);
|
||||||
let mut reader = decoder
|
let mut reader = decoder
|
||||||
.read_info()
|
.read_info()
|
||||||
@@ -286,8 +287,8 @@ fn prepare_png(file: NamedTempFile, bytes: Vec<u8>) -> Result<PhotoPrep, String>
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// JPEG branch: zune-jpeg decode → Lanczos downscale → jpeg-encoder output.
|
/// JPEG branch: zune-jpeg decode → Lanczos downscale → jpeg-encoder output.
|
||||||
fn prepare_jpeg(file: NamedTempFile, bytes: Vec<u8>) -> Result<PhotoPrep, String> {
|
fn prepare_jpeg(file: NamedTempFile, bytes: &[u8]) -> Result<PhotoPrep, String> {
|
||||||
let mut decoder = zune_jpeg::JpegDecoder::new(std::io::Cursor::new(&bytes));
|
let mut decoder = zune_jpeg::JpegDecoder::new(std::io::Cursor::new(bytes));
|
||||||
// Decodes to RGB by default. Headers first so dimensions are known before
|
// Decodes to RGB by default. Headers first so dimensions are known before
|
||||||
// the (potentially huge) pixel decode.
|
// the (potentially huge) pixel decode.
|
||||||
decoder
|
decoder
|
||||||
@@ -423,7 +424,7 @@ mod tests {
|
|||||||
|
|
||||||
let mut file = tempfile::Builder::new().suffix(".png").tempfile().unwrap();
|
let mut file = tempfile::Builder::new().suffix(".png").tempfile().unwrap();
|
||||||
std::io::Write::write_all(file.as_file_mut(), &bytes).unwrap();
|
std::io::Write::write_all(file.as_file_mut(), &bytes).unwrap();
|
||||||
prepare_photo(file)
|
prepare_photo(file, &bytes)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -468,7 +469,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
let mut file = tempfile::Builder::new().suffix(".jpg").tempfile().unwrap();
|
let mut file = tempfile::Builder::new().suffix(".jpg").tempfile().unwrap();
|
||||||
std::io::Write::write_all(file.as_file_mut(), &bytes).unwrap();
|
std::io::Write::write_all(file.as_file_mut(), &bytes).unwrap();
|
||||||
match prepare_photo(file).unwrap() {
|
match prepare_photo(file, &bytes).unwrap() {
|
||||||
PhotoPrep::Upload(file) => {
|
PhotoPrep::Upload(file) => {
|
||||||
let out = std::fs::read(file.path()).unwrap();
|
let out = std::fs::read(file.path()).unwrap();
|
||||||
assert!(out.starts_with(&[0xFF, 0xD8]), "output must stay jpeg");
|
assert!(out.starts_with(&[0xFF, 0xD8]), "output must stay jpeg");
|
||||||
@@ -516,7 +517,7 @@ mod tests {
|
|||||||
|
|
||||||
let mut file = tempfile::Builder::new().suffix(".png").tempfile().unwrap();
|
let mut file = tempfile::Builder::new().suffix(".png").tempfile().unwrap();
|
||||||
std::io::Write::write_all(file.as_file_mut(), &bytes).unwrap();
|
std::io::Write::write_all(file.as_file_mut(), &bytes).unwrap();
|
||||||
match prepare_photo(file).unwrap() {
|
match prepare_photo(file, &bytes).unwrap() {
|
||||||
PhotoPrep::Upload(file) => {
|
PhotoPrep::Upload(file) => {
|
||||||
let out = std::fs::read(file.path()).unwrap();
|
let out = std::fs::read(file.path()).unwrap();
|
||||||
assert!(out.starts_with(&[0xFF, 0xD8]), "must transcode to JPEG");
|
assert!(out.starts_with(&[0xFF, 0xD8]), "must transcode to JPEG");
|
||||||
|
|||||||
@@ -556,9 +556,14 @@ enum FallbackError {
|
|||||||
/// only if still too big. Anything that cannot be fixed falls back to the
|
/// only if still too big. Anything that cannot be fixed falls back to the
|
||||||
/// item's smaller URL.
|
/// item's smaller URL.
|
||||||
///
|
///
|
||||||
/// Downloads one media item to a temp file (deleted on drop). Network errors
|
/// Downloads one media item to a temp file (deleted on drop), returning the
|
||||||
/// are retryable; size over the upload cap and other download errors are not.
|
/// file plus the downloaded bytes (photos keep the bytes for
|
||||||
async fn download_to_temp(item: &MediaItemPayload) -> Result<NamedTempFile, FallbackError> {
|
/// [`photo::prepare_photo`] — re-reading the file would double the I/O).
|
||||||
|
/// Network errors are retryable; size over the upload cap and other download
|
||||||
|
/// errors are not.
|
||||||
|
async fn download_to_temp(
|
||||||
|
item: &MediaItemPayload,
|
||||||
|
) -> Result<(NamedTempFile, bytes::Bytes), FallbackError> {
|
||||||
let media_url = match item {
|
let media_url = match item {
|
||||||
MediaItemPayload::Photo { media, .. }
|
MediaItemPayload::Photo { media, .. }
|
||||||
| MediaItemPayload::Video { media, .. }
|
| MediaItemPayload::Video { media, .. }
|
||||||
@@ -601,7 +606,7 @@ async fn download_to_temp(item: &MediaItemPayload) -> Result<NamedTempFile, Fall
|
|||||||
.map_err(|e| FallbackError::Permanent {
|
.map_err(|e| FallbackError::Permanent {
|
||||||
message: format!("temp file write failed: {e}"),
|
message: format!("temp file write failed: {e}"),
|
||||||
})?;
|
})?;
|
||||||
Ok(file)
|
Ok((file, bytes))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Builds the media group item from an uploaded file.
|
/// Builds the media group item from an uploaded file.
|
||||||
@@ -711,14 +716,14 @@ async fn prepare_upload_item(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
match download_to_temp(&item).await {
|
match download_to_temp(&item).await {
|
||||||
Ok(file) => {
|
Ok((file, bytes)) => {
|
||||||
if matches!(item, MediaItemPayload::Photo { .. }) {
|
if matches!(item, MediaItemPayload::Photo { .. }) {
|
||||||
// Telegram rejects photos wider+taller than 10000 px combined
|
// Telegram rejects photos wider+taller than 10000 px combined
|
||||||
// (PHOTO_INVALID_DIMENSIONS): downscale the downloaded file
|
// (PHOTO_INVALID_DIMENSIONS): downscale the downloaded file
|
||||||
// before uploading; photos that cannot be brought within the
|
// before uploading; photos that cannot be brought within the
|
||||||
// limits degrade to the smaller URL. CPU-heavy work runs off
|
// limits degrade to the smaller URL. CPU-heavy work runs off
|
||||||
// the async executor thread.
|
// the async executor thread.
|
||||||
let prep = tokio::task::spawn_blocking(move || photo::prepare_photo(file))
|
let prep = tokio::task::spawn_blocking(move || photo::prepare_photo(file, &bytes))
|
||||||
.await
|
.await
|
||||||
.map_err(|e| FallbackError::Permanent {
|
.map_err(|e| FallbackError::Permanent {
|
||||||
message: format!("photo worker panicked: {e}"),
|
message: format!("photo worker panicked: {e}"),
|
||||||
@@ -1047,7 +1052,7 @@ pub async fn send_animation(bot: &Bot, task: &Task) -> Result<Vec<i64>, SendErro
|
|||||||
media_url
|
media_url
|
||||||
);
|
);
|
||||||
match download_to_temp(animation).await {
|
match download_to_temp(animation).await {
|
||||||
Ok(file) => {
|
Ok((file, _bytes)) => {
|
||||||
let path = file.path().to_path_buf();
|
let path = file.path().to_path_buf();
|
||||||
match send_animation_inner(
|
match send_animation_inner(
|
||||||
bot,
|
bot,
|
||||||
|
|||||||
Reference in New Issue
Block a user