fix: re-fetch queued retries whose local media did not survive a restart

A queued retry that holds a local file — the ugoira MP4, a bsky remux, or a
temp file the reupload fallback downloaded — could never succeed after a
restart: those files live in the system temp dir and `send::KEEP_ALIVE`, the
registry that keeps them alive for the retry, is in memory. The row retried
into an upload error, said nothing about why, and dead-lettered the user's
link even though the payload carries the `source_url`.

`handlers::repair_lost_local_media` now runs in `main` before any worker
starts (so no row can be leased while it writes payloads, which is why it can
replace them without the lease guard a worker's write-back carries):

- `Task::local_media_paths` decides which rows are affected: any local path
  that is gone. A partially delivered album is left alone — its remaining
  batches cannot be reconciled with a fresh media list without risking a
  second copy of what the user already received.
- The post is re-fetched from `source_url` through the ordinary `site::fetch`,
  so a repaired task looks like a first send: fresh media, the chat's caption
  format, a fresh link-cache snapshot, and a new keep-alive entry when the
  re-fetch produced another local file.
- The delivery envelope (chat, reply, forward/edit settings, notify targets) is
  kept, the attempt budget restarts, and nothing counts as sent.
- A post that cannot be fetched again (gone, withheld, site down) notifies the
  user with that reason instead of letting the retry die on a missing file.

New queue plumbing: `runnable_rows()` (pending + in-progress rows, read before
the workers exist) and `replace_payload()` (rewrites the payload, resets
`attempts`, marks the row pending).

Verified: 5 new offline tests (the two decisions above against a real temp
file, the queue scan/replace, and the envelope-preserving rewrite) plus
`a_lost_local_media_row_is_refetched_from_its_post`, a live test that seeds a
row pointing at a missing file with a real bsky post as its source and asserts
the row now carries http(s) media and that nothing was sent — run against the
live API here. `cargo fmt`, `cargo clippy --workspace --all-targets --locked --
-D warnings` and `cargo test --workspace --locked` (187 passed, 15 ignored)
are clean.
This commit is contained in:
2026-09-21 01:48:50 +08:00
parent 024dfd50b3
commit d540fc31e9
6 changed files with 531 additions and 4 deletions
+100
View File
@@ -235,6 +235,61 @@ impl PersistentTaskQueue {
/// `earliest_run_after`: that one runs on every idle worker cycle and must
/// stay a single indexed `MIN`, while the count is only asked for once per
/// sweep.
/// `(id, payload)` of every row that can still run (`pending`,
/// `in_progress`). The startup repair reads these before the workers start:
/// with no worker running, no row can be leased while it writes.
pub async fn runnable_rows(&self) -> Vec<(String, String)> {
let result = self
.pool
.with_conn(|conn| {
let mut stmt = conn.prepare(
"SELECT id, payload FROM tasks WHERE status IN ('pending', 'in_progress') ORDER BY run_after",
)?;
let rows = stmt.query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?;
rows.collect::<rusqlite::Result<Vec<(String, String)>>>()
})
.await;
match result {
Ok(rows) => rows,
Err(e) => {
log::error!("queue row scan failed: {e}");
Vec::new()
}
}
}
/// Replaces a runnable row's payload and restarts its attempt budget: the
/// new payload is a fresh delivery of the same task, so the retries it has
/// already spent do not carry over. Startup repair only — a worker's
/// write-back is lease-token guarded instead (`replace_payload` cannot race
/// one: it runs before any worker does).
pub async fn replace_payload(&self, id: &str, payload: &Value) -> bool {
let logged_id = id.to_string();
let (id, payload) = (id.to_string(), payload.to_string());
let result = self
.pool
.with_conn(move |conn| {
let affected = conn.execute(
"UPDATE tasks SET payload=?1, attempts=0, run_after=?2, status='pending', locked_until=0 \
WHERE id=?3 AND status IN ('pending', 'in_progress')",
params![payload, now_f64(), id],
)?;
Ok(affected == 1)
})
.await;
match result {
Ok(true) => true,
Ok(false) => {
log::warn!("queue: row {logged_id} vanished before its payload could be replaced");
false
}
Err(e) => {
log::error!("queue payload replace failed for {logged_id}: {e}");
false
}
}
}
pub async fn pending_backlog(&self) -> Option<(i64, f64)> {
let result = self
.pool
@@ -894,6 +949,51 @@ mod tests {
queue.stop().await;
}
#[tokio::test]
async fn runnable_rows_and_payload_replacement() {
let (queue, _dir) = new_queue().await;
queue
.enqueue(serde_json::json!({"s": 1}), now_f64())
.await
.unwrap();
let rows = queue.runnable_rows().await;
assert_eq!(rows.len(), 1);
let (id, payload) = rows[0].clone();
assert_eq!(payload, "{\"s\":1}");
// A replacement restarts the attempt budget (the new payload is a fresh
// delivery, not the continuation of the old one).
{
let id_owned = id.clone();
queue
.pool
.with_conn(move |conn| {
conn.execute("UPDATE tasks SET attempts=2 WHERE id=?1", params![id_owned])?;
Ok(())
})
.await
.unwrap();
}
assert!(
queue
.replace_payload(&id, &serde_json::json!({"s": 2}))
.await
);
let rows = queue.runnable_rows().await;
assert_eq!(rows[0].1, "{\"s\":2}");
assert_eq!(
queue.pending_backlog().await.map(|(n, _)| n),
Some(1),
"a repaired row is pending work again"
);
// A row that is gone (or done) is not rewritten.
assert!(
!queue
.replace_payload("task_missing", &serde_json::json!({}))
.await
);
}
#[tokio::test]
async fn permanent_error_dead_letters_immediately() {
let (queue, _dir) = new_queue().await;