queue: back off 1s when a lease fails

A lease error (e.g. persistent SQLITE_BUSY) while rows are due made the
worker spin with sleep(0), hammering SQLite and flooding the log.
lease_next now returns the error and the loop sleeps 1s before retrying.
This commit is contained in:
2026-08-08 20:09:04 +08:00
parent 51cc079a85
commit f40639c799
+12 -12
View File
@@ -240,8 +240,8 @@ impl QueueWorker {
async fn run_loop(self) { async fn run_loop(self) {
while !self.stop.load(Ordering::Relaxed) { while !self.stop.load(Ordering::Relaxed) {
match self.lease_next().await { match self.lease_next().await {
Some(row) => self.process(row).await, Ok(Some(row)) => self.process(row).await,
None => { Ok(None) => {
let wait_until = self.earliest_run_after().await; let wait_until = self.earliest_run_after().await;
let notified = self.notify.notified(); let notified = self.notify.notified();
tokio::pin!(notified); tokio::pin!(notified);
@@ -258,13 +258,20 @@ impl QueueWorker {
} }
} }
} }
// A lease failure while rows are due would otherwise loop
// with sleep(0) and hammer SQLite; back off briefly.
Err(e) => {
log::error!("queue lease failed: {e}");
tokio::time::sleep(Duration::from_secs(1)).await;
}
} }
} }
} }
/// Leases the oldest due row (sets it `in_progress` with a lock TTL). /// Leases the oldest due row (sets it `in_progress` with a lock TTL).
async fn lease_next(&self) -> Option<LeasedRow> { /// Errors are surfaced so the caller can back off instead of spinning.
let result = crate::db::with_conn(&self.db_path, |conn| { async fn lease_next(&self) -> Result<Option<LeasedRow>, rusqlite::Error> {
crate::db::with_conn(&self.db_path, |conn| {
// BEGIN IMMEDIATE: with several workers, a deferred transaction // BEGIN IMMEDIATE: with several workers, a deferred transaction
// that read before another worker's lease commit would fail with // that read before another worker's lease commit would fail with
// SQLITE_BUSY_SNAPSHOT. Taking the write lock up front serializes // SQLITE_BUSY_SNAPSHOT. Taking the write lock up front serializes
@@ -302,14 +309,7 @@ impl QueueWorker {
attempts, attempts,
})) }))
}) })
.await; .await
match result {
Ok(row) => row,
Err(e) => {
log::error!("queue lease failed: {e}");
None
}
}
} }
async fn earliest_run_after(&self) -> Option<f64> { async fn earliest_run_after(&self) -> Option<f64> {