mirror of
https://github.com/TheFunny/TelegramTwitterMediaBot.git
synced 2026-09-23 23:32:05 +00:00
fix(send): keep local media (ugoira/bsky remux MP4) alive across queue retries
A task whose media is a locally produced file (pixiv ugoira MP4, bsky HLS
remux MP4) references a path inside a tempfile TempDir owned by Fetched.
The retry ran after that TempDir was dropped, so the file was already gone
and the retry always failed permanently ("local media file missing") — and
the upload fallback even tried to GET the local path as a URL.
Keep the temp dirs in a process-wide registry (Fetched::take_keep_alive ->
send::KEEP_ALIVE) that is only released when the task settles (sent or
permanently failed); the upload fallback now uploads local files directly
instead of attempting to download them.
Restart-mid-queue still loses the files (documented behavior in
input_file_for) — only the in-process retry path is fixed here.
This commit is contained in:
@@ -95,6 +95,14 @@ impl Fetched {
|
|||||||
)
|
)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Hands over the temp dir keeping locally produced media (ugoira MP4,
|
||||||
|
/// bsky remux MP4) alive. The bot keeps it while its task may still be
|
||||||
|
/// retried by the queue, which runs after this [`Fetched`] is dropped and
|
||||||
|
/// its temp files would otherwise be gone. `None` when no such dir exists.
|
||||||
|
pub fn take_keep_alive(&mut self) -> Option<tempfile::TempDir> {
|
||||||
|
self._keep_alive.take()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Renders a user-supplied caption format from raw (already-escaped) field
|
/// Renders a user-supplied caption format from raw (already-escaped) field
|
||||||
|
|||||||
@@ -516,6 +516,8 @@ async fn dispatch_send(bot: Bot, message: &Message, task: &Task, url: &str) {
|
|||||||
Ok(message_ids) => {
|
Ok(message_ids) => {
|
||||||
log::info!("sent {} message(s) for {url}", message_ids.len());
|
log::info!("sent {} message(s) for {url}", message_ids.len());
|
||||||
send::post_send_actions(&bot, task, message_ids).await;
|
send::post_send_actions(&bot, task, message_ids).await;
|
||||||
|
// The task settled: drop any keep-alive temp media.
|
||||||
|
send::release_keep_alive(task);
|
||||||
}
|
}
|
||||||
Err(send::SendError::Retryable {
|
Err(send::SendError::Retryable {
|
||||||
delay_seconds,
|
delay_seconds,
|
||||||
@@ -530,6 +532,7 @@ async fn dispatch_send(bot: Bot, message: &Message, task: &Task, url: &str) {
|
|||||||
task,
|
task,
|
||||||
}) => {
|
}) => {
|
||||||
send::invalidate_cache(&task).await;
|
send::invalidate_cache(&task).await;
|
||||||
|
send::release_keep_alive(&task);
|
||||||
log::error!("send for {url} failed permanently: {err_message}");
|
log::error!("send for {url} failed permanently: {err_message}");
|
||||||
let _ = reply(bot, message.clone(), format!("Send failed: {err_message}")).await;
|
let _ = reply(bot, message.clone(), format!("Send failed: {err_message}")).await;
|
||||||
}
|
}
|
||||||
@@ -667,7 +670,7 @@ async fn url_media(bot: Bot, message: &Message, url: &str) {
|
|||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
Ok(Some(fetched)) => {
|
Ok(Some(mut fetched)) => {
|
||||||
if fetched.media.is_empty() {
|
if fetched.media.is_empty() {
|
||||||
let _ = reply(
|
let _ = reply(
|
||||||
bot,
|
bot,
|
||||||
@@ -712,6 +715,13 @@ async fn url_media(bot: Bot, message: &Message, url: &str) {
|
|||||||
items,
|
items,
|
||||||
cache_data,
|
cache_data,
|
||||||
);
|
);
|
||||||
|
// Hand the keep-alive temp dir (ugoira / bsky remux MP4) to the
|
||||||
|
// retry registry: a queued retry runs after this function returns
|
||||||
|
// and the fetch's own TempDir is dropped, so without this the
|
||||||
|
// local file would be gone by the time the retry sends it.
|
||||||
|
if let Some(dir) = fetched.take_keep_alive() {
|
||||||
|
send::KEEP_ALIVE.lock().push(dir);
|
||||||
|
}
|
||||||
dispatch_send(bot, message, &task, url).await;
|
dispatch_send(bot, message, &task, url).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -147,6 +147,39 @@ impl Task {
|
|||||||
fn is_cached_send(&self) -> bool {
|
fn is_cached_send(&self) -> bool {
|
||||||
self.cache_data().is_some_and(|c| !c.media.is_empty())
|
self.cache_data().is_some_and(|c| !c.media.is_empty())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// All media payloads of this task (sequence batches flattened plus the
|
||||||
|
/// lone animation).
|
||||||
|
fn media_items(&self) -> Vec<&MediaItemPayload> {
|
||||||
|
match self {
|
||||||
|
Task::SendMediaSequence { media_batches, .. } => {
|
||||||
|
media_batches.iter().flatten().collect()
|
||||||
|
}
|
||||||
|
Task::SendAnimation { animation, .. } => std::slice::from_ref(animation).iter().collect(),
|
||||||
|
Task::ForwardMessages { .. } => Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Local file paths referenced by this task's media (ugoira / bsky remux
|
||||||
|
/// MP4 and the like); empty for URL or Telegram file-id sends.
|
||||||
|
fn local_media_paths(&self) -> Vec<std::path::PathBuf> {
|
||||||
|
let mut out = Vec::new();
|
||||||
|
for item in self.media_items() {
|
||||||
|
let is_file_id = match item {
|
||||||
|
MediaItemPayload::Photo { file_id, .. }
|
||||||
|
| MediaItemPayload::Video { file_id, .. }
|
||||||
|
| MediaItemPayload::Animation { file_id, .. } => *file_id,
|
||||||
|
};
|
||||||
|
if is_file_id {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let media = item_url(item);
|
||||||
|
if !media.starts_with("http://") && !media.starts_with("https://") {
|
||||||
|
out.push(std::path::PathBuf::from(media));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Telegram file id of the message's media, matched to the payload kind.
|
/// Telegram file id of the message's media, matched to the payload kind.
|
||||||
@@ -227,6 +260,30 @@ pub async fn invalidate_cache(task: &Task) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Locally produced media files (ugoira MP4, bsky remux MP4) whose temp dirs
|
||||||
|
/// must stay alive while their task may be retried by the queue. The fetch
|
||||||
|
/// pipeline hands ownership here via [`x_media::site::Fetched::take_keep_alive`]
|
||||||
|
/// before the [`Fetched`] is dropped; a queued retry runs after that drop, so
|
||||||
|
/// without this the local file would be gone by the time the retry sends it.
|
||||||
|
/// Entries are removed when the task settles (see [`release_keep_alive`]).
|
||||||
|
pub static KEEP_ALIVE: LazyLock<parking_lot::Mutex<Vec<tempfile::TempDir>>> =
|
||||||
|
LazyLock::new(|| parking_lot::Mutex::new(Vec::new()));
|
||||||
|
|
||||||
|
/// Drops the keep-alive temp dirs holding media referenced by `task` (matched
|
||||||
|
/// by path prefix). Called once a task settles — sent or permanently failed —
|
||||||
|
/// so retry-only temp files do not leak; retryable tasks keep them alive.
|
||||||
|
pub fn release_keep_alive(task: &Task) {
|
||||||
|
let paths = task.local_media_paths();
|
||||||
|
if paths.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let mut alive = KEEP_ALIVE.lock();
|
||||||
|
alive.retain(|dir| {
|
||||||
|
let dir_path = dir.path();
|
||||||
|
!paths.iter().any(|p| p.starts_with(dir_path))
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
pub const MAX_MEDIA_GROUP: usize = 9;
|
pub const MAX_MEDIA_GROUP: usize = 9;
|
||||||
|
|
||||||
/// Splits media into batches of at most [`MAX_MEDIA_GROUP`] items.
|
/// Splits media into batches of at most [`MAX_MEDIA_GROUP`] items.
|
||||||
@@ -595,6 +652,18 @@ async fn send_batch_via_upload(
|
|||||||
let mut items = Vec::new();
|
let mut items = Vec::new();
|
||||||
for (i, item) in batch.iter().enumerate() {
|
for (i, item) in batch.iter().enumerate() {
|
||||||
let item_caption = if i == 0 { caption } else { None };
|
let item_caption = if i == 0 { caption } else { None };
|
||||||
|
// Locally produced files (ugoira / bsky remux MP4): nothing to
|
||||||
|
// download or shrink — upload the file directly. The send is a
|
||||||
|
// multipart upload, so the only remaining failure is an upload-cap
|
||||||
|
// error, which is permanent (a video cannot be re-encoded here).
|
||||||
|
let media_url = item_url(item);
|
||||||
|
if !media_url.starts_with("http://") && !media_url.starts_with("https://") {
|
||||||
|
let path = std::path::PathBuf::from(media_url);
|
||||||
|
let media = media_from_file(item, path, item_caption, item.thumbnail_url())
|
||||||
|
.map_err(|message| FallbackError::Permanent { message })?;
|
||||||
|
items.push(media);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
// Size check before downloading/uploading: over the cap, use the
|
// Size check before downloading/uploading: over the cap, use the
|
||||||
// smaller URL instead of the file. Photos are exempt — they are
|
// smaller URL instead of the file. Photos are exempt — they are
|
||||||
// downloaded and processed (downscale / PNG→JPEG) before uploading.
|
// downloaded and processed (downscale / PNG→JPEG) before uploading.
|
||||||
@@ -1199,6 +1268,8 @@ pub async fn handle_task(payload: serde_json::Value) -> Result<(), QueueError> {
|
|||||||
}
|
}
|
||||||
Err(SendError::Permanent { message, task }) => {
|
Err(SendError::Permanent { message, task }) => {
|
||||||
invalidate_cache(&task).await;
|
invalidate_cache(&task).await;
|
||||||
|
// The task settles here: drop any keep-alive temp media.
|
||||||
|
release_keep_alive(&task);
|
||||||
return Err(QueueError::Permanent {
|
return Err(QueueError::Permanent {
|
||||||
message,
|
message,
|
||||||
payload: serde_json::to_value(task).expect("task serializes"),
|
payload: serde_json::to_value(task).expect("task serializes"),
|
||||||
@@ -1208,6 +1279,7 @@ pub async fn handle_task(payload: serde_json::Value) -> Result<(), QueueError> {
|
|||||||
if !resumed {
|
if !resumed {
|
||||||
post_send_actions(&bot, &task, message_ids).await;
|
post_send_actions(&bot, &task, message_ids).await;
|
||||||
}
|
}
|
||||||
|
release_keep_alive(&task);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
Task::ForwardMessages { .. } => match forward_messages(&bot, &task).await {
|
Task::ForwardMessages { .. } => match forward_messages(&bot, &task).await {
|
||||||
@@ -1219,10 +1291,13 @@ pub async fn handle_task(payload: serde_json::Value) -> Result<(), QueueError> {
|
|||||||
delay_seconds,
|
delay_seconds,
|
||||||
payload: serde_json::to_value(task).expect("task serializes"),
|
payload: serde_json::to_value(task).expect("task serializes"),
|
||||||
}),
|
}),
|
||||||
Err(SendError::Permanent { message, task }) => Err(QueueError::Permanent {
|
Err(SendError::Permanent { message, task }) => {
|
||||||
|
release_keep_alive(&task);
|
||||||
|
Err(QueueError::Permanent {
|
||||||
message,
|
message,
|
||||||
payload: serde_json::to_value(task).expect("task serializes"),
|
payload: serde_json::to_value(task).expect("task serializes"),
|
||||||
}),
|
})
|
||||||
|
}
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user