mirror of
https://github.com/TheFunny/TelegramTwitterMediaBot.git
synced 2026-09-23 23:32:05 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
020e2d01a3
|
||
|
|
3d6f8548c3
|
||
|
|
063e910473
|
||
|
|
b0ced34b4c
|
||
|
|
4060a88031
|
||
|
|
fb43441c56
|
||
|
|
bf628dc999
|
||
|
|
de22aa9b4d
|
Generated
+2
-2
@@ -3351,7 +3351,7 @@ checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "x-media"
|
name = "x-media"
|
||||||
version = "1.0.7"
|
version = "1.0.8"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytes",
|
"bytes",
|
||||||
"dotenv",
|
"dotenv",
|
||||||
@@ -3370,7 +3370,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "xmedia-bot"
|
name = "xmedia-bot"
|
||||||
version = "1.0.7"
|
version = "1.0.8"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"dotenv",
|
"dotenv",
|
||||||
"fast_image_resize",
|
"fast_image_resize",
|
||||||
|
|||||||
@@ -101,12 +101,14 @@ Telegram 只接受 443/80/88/8443 端口。
|
|||||||
|
|
||||||
| 命令 | 说明 |
|
| 命令 | 说明 |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `/set_forward_channel <频道>` | 设置转发频道 |
|
| `/start` | 欢迎语 |
|
||||||
|
| `/help` | 查看全部命令及用法(即本文档的命令表) |
|
||||||
|
| `/set_forward_channel <频道>` | 设置转发频道,参数为 `@频道名` 或频道 ID;设置后发送的媒体消息会自动转发到该频道 |
|
||||||
| `/remove_forward_channel` | 取消转发频道 |
|
| `/remove_forward_channel` | 取消转发频道 |
|
||||||
| `/edit_before_forward` | 开关转发前编辑 |
|
| `/edit_before_forward` | 开关「转发前编辑」:开启后,转发成功后 bot 会发一条提示消息,回复它可修改第一条转发消息的 caption(或点击模板按钮套用模板) |
|
||||||
| `/set_template <名称>` | 将回复的消息(含 `[]`)保存为模板 |
|
| `/set_template <名称>` | 回复一条含 `[]` 的消息,将其保存为命名模板;转发时 `[]` 会被替换为原帖链接(配合「转发前编辑」使用) |
|
||||||
| `/set_format <站点> <格式>` | 自定义 caption 格式(占位符 `{url}` `{title}` `{tags}` 等) |
|
| `/set_format <站点> <格式>` | 自定义某站点的 caption 格式。站点:`twitter` / `bsky` / `pixiv`。占位符:`{url}` `{author}` `{author_url}` `{title}` `{tags}` |
|
||||||
| `/bot_dict` | 查看聊天状态 |
|
| `/bot_dict` | 查看当前聊天状态(调试用) |
|
||||||
|
|
||||||
链接处理仅限私聊;命令在任意聊天可用。
|
链接处理仅限私聊;命令在任意聊天可用。
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "x-media"
|
name = "x-media"
|
||||||
version = "1.0.7"
|
version = "1.0.8"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
|
|||||||
@@ -5,9 +5,8 @@ use html_escape::encode_text;
|
|||||||
use regex::Regex;
|
use regex::Regex;
|
||||||
use std::sync::LazyLock;
|
use std::sync::LazyLock;
|
||||||
|
|
||||||
pub static PATTERN: LazyLock<Regex> = LazyLock::new(|| {
|
pub static PATTERN: LazyLock<Regex> =
|
||||||
Regex::new(r"bsky\.app/profile/([\w.\-:]+)/post/([\w.\-~]+)").unwrap()
|
LazyLock::new(|| Regex::new(r"bsky\.app/profile/([\w.\-:]+)/post/([\w.\-~]+)").unwrap());
|
||||||
});
|
|
||||||
|
|
||||||
pub fn enabled() -> bool {
|
pub fn enabled() -> bool {
|
||||||
true
|
true
|
||||||
@@ -15,8 +14,14 @@ pub fn enabled() -> bool {
|
|||||||
|
|
||||||
pub async fn fetch_from_url(url: &str) -> Result<Fetched, FetchError> {
|
pub async fn fetch_from_url(url: &str) -> Result<Fetched, FetchError> {
|
||||||
let caps = PATTERN.captures(url).ok_or(FetchError::NotFound)?;
|
let caps = PATTERN.captures(url).ok_or(FetchError::NotFound)?;
|
||||||
let handle = caps.get(1).map(|m| m.as_str()).ok_or(FetchError::NotFound)?;
|
let handle = caps
|
||||||
let rkey = caps.get(2).map(|m| m.as_str()).ok_or(FetchError::NotFound)?;
|
.get(1)
|
||||||
|
.map(|m| m.as_str())
|
||||||
|
.ok_or(FetchError::NotFound)?;
|
||||||
|
let rkey = caps
|
||||||
|
.get(2)
|
||||||
|
.map(|m| m.as_str())
|
||||||
|
.ok_or(FetchError::NotFound)?;
|
||||||
Ok(fetch(handle, rkey).await?.into())
|
Ok(fetch(handle, rkey).await?.into())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -197,7 +202,10 @@ mod tests {
|
|||||||
}));
|
}));
|
||||||
let post = Post::from_json(&raw.to_string(), "3xxxx".into()).unwrap();
|
let post = Post::from_json(&raw.to_string(), "3xxxx".into()).unwrap();
|
||||||
let fetched: Fetched = post.into();
|
let fetched: Fetched = post.into();
|
||||||
assert_eq!(fetched.source_url, "https://bsky.app/profile/user.bsky.social/post/3xxxx");
|
assert_eq!(
|
||||||
|
fetched.source_url,
|
||||||
|
"https://bsky.app/profile/user.bsky.social/post/3xxxx"
|
||||||
|
);
|
||||||
assert_eq!(fetched.title, "hello <world>");
|
assert_eq!(fetched.title, "hello <world>");
|
||||||
assert_eq!(fetched.media.len(), 1);
|
assert_eq!(fetched.media.len(), 1);
|
||||||
assert!(!fetched.sensitive);
|
assert!(!fetched.sensitive);
|
||||||
@@ -246,11 +254,10 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn live_fetch_with_photos() {
|
async fn live_fetch_with_photos() {
|
||||||
let fetched = fetch_from_url(
|
let fetched =
|
||||||
"https://bsky.app/profile/asagi0398.bsky.social/post/3mqkhrq5w6k2m",
|
fetch_from_url("https://bsky.app/profile/asagi0398.bsky.social/post/3mqkhrq5w6k2m")
|
||||||
)
|
.await
|
||||||
.await
|
.unwrap();
|
||||||
.unwrap();
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
fetched.source_url,
|
fetched.source_url,
|
||||||
"https://bsky.app/profile/asagi0398.bsky.social/post/3mqkhrq5w6k2m"
|
"https://bsky.app/profile/asagi0398.bsky.social/post/3mqkhrq5w6k2m"
|
||||||
@@ -260,9 +267,10 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn live_fetch_smoke() {
|
async fn live_fetch_smoke() {
|
||||||
let fetched = fetch_from_url("https://bsky.app/profile/fu-futa.bsky.social/post/3laoveufjv224")
|
let fetched =
|
||||||
.await
|
fetch_from_url("https://bsky.app/profile/fu-futa.bsky.social/post/3laoveufjv224")
|
||||||
.unwrap();
|
.await
|
||||||
|
.unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
fetched.source_url,
|
fetched.source_url,
|
||||||
"https://bsky.app/profile/fu-futa.bsky.social/post/3laoveufjv224"
|
"https://bsky.app/profile/fu-futa.bsky.social/post/3laoveufjv224"
|
||||||
|
|||||||
@@ -179,7 +179,6 @@ impl From<reqwest::Error> for FetchError {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
impl From<serde_json::Error> for FetchError {
|
impl From<serde_json::Error> for FetchError {
|
||||||
fn from(e: serde_json::Error) -> Self {
|
fn from(e: serde_json::Error) -> Self {
|
||||||
FetchError::Json(e)
|
FetchError::Json(e)
|
||||||
|
|||||||
@@ -10,8 +10,8 @@ use crate::site::FetchError;
|
|||||||
use std::env;
|
use std::env;
|
||||||
use std::fmt;
|
use std::fmt;
|
||||||
use std::io::{Cursor, Read};
|
use std::io::{Cursor, Read};
|
||||||
use std::sync::atomic::{AtomicBool, Ordering};
|
|
||||||
use std::sync::LazyLock;
|
use std::sync::LazyLock;
|
||||||
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
use std::time::{Duration, SystemTime};
|
use std::time::{Duration, SystemTime};
|
||||||
|
|
||||||
const AUTH_TOKEN_URL: &str = "https://oauth.secure.pixiv.net/auth/token";
|
const AUTH_TOKEN_URL: &str = "https://oauth.secure.pixiv.net/auth/token";
|
||||||
@@ -127,7 +127,9 @@ impl PixivAPI {
|
|||||||
pub async fn illust_detail(&self, illust_id: u64) -> Result<IllustrationModel, PixivError> {
|
pub async fn illust_detail(&self, illust_id: u64) -> Result<IllustrationModel, PixivError> {
|
||||||
let access_token = self.get_access_token().await?;
|
let access_token = self.get_access_token().await?;
|
||||||
let response = crate::site::CLIENT
|
let response = crate::site::CLIENT
|
||||||
.get(format!("{APP_API_URL}/v1/illust/detail?illust_id={illust_id}"))
|
.get(format!(
|
||||||
|
"{APP_API_URL}/v1/illust/detail?illust_id={illust_id}"
|
||||||
|
))
|
||||||
.header("app-os", "ios")
|
.header("app-os", "ios")
|
||||||
.header("app-os-version", "14.6")
|
.header("app-os-version", "14.6")
|
||||||
.header("User-Agent", APP_USER_AGENT)
|
.header("User-Agent", APP_USER_AGENT)
|
||||||
@@ -175,7 +177,9 @@ impl PixivAPI {
|
|||||||
pub async fn ugoira_metadata(&self, illust_id: u64) -> Result<UgoiraMetadataModel, PixivError> {
|
pub async fn ugoira_metadata(&self, illust_id: u64) -> Result<UgoiraMetadataModel, PixivError> {
|
||||||
let access_token = self.get_access_token().await?;
|
let access_token = self.get_access_token().await?;
|
||||||
let response = crate::site::CLIENT
|
let response = crate::site::CLIENT
|
||||||
.get(format!("{APP_API_URL}/v1/ugoira/metadata?illust_id={illust_id}"))
|
.get(format!(
|
||||||
|
"{APP_API_URL}/v1/ugoira/metadata?illust_id={illust_id}"
|
||||||
|
))
|
||||||
.header("app-os", "ios")
|
.header("app-os", "ios")
|
||||||
.header("app-os-version", "14.6")
|
.header("app-os-version", "14.6")
|
||||||
.header("User-Agent", APP_USER_AGENT)
|
.header("User-Agent", APP_USER_AGENT)
|
||||||
@@ -218,87 +222,89 @@ impl PixivAPI {
|
|||||||
let Some(zip_url) = zip_url else {
|
let Some(zip_url) = zip_url else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
let zip_bytes = crate::site::download_media(&zip_url).await.map_err(|e| match e {
|
let zip_bytes = crate::site::download_media(&zip_url)
|
||||||
FetchError::Http(e) => PixivError::Http(e),
|
.await
|
||||||
other => PixivError::Api(format!("frame zip download failed: {other}")),
|
.map_err(|e| match e {
|
||||||
})?;
|
FetchError::Http(e) => PixivError::Http(e),
|
||||||
|
other => PixivError::Api(format!("frame zip download failed: {other}")),
|
||||||
|
})?;
|
||||||
let frame_delays = metadata.frames.iter().map(|f| f.delay).collect::<Vec<_>>();
|
let frame_delays = metadata.frames.iter().map(|f| f.delay).collect::<Vec<_>>();
|
||||||
let result = tokio::task::spawn_blocking(
|
let result =
|
||||||
move || -> Result<(String, tempfile::TempDir), String> {
|
tokio::task::spawn_blocking(move || -> Result<(String, tempfile::TempDir), String> {
|
||||||
let frames_dir = tempfile::tempdir().map_err(|e| e.to_string())?;
|
let frames_dir = tempfile::tempdir().map_err(|e| e.to_string())?;
|
||||||
let out_dir = tempfile::tempdir().map_err(|e| e.to_string())?;
|
let out_dir = tempfile::tempdir().map_err(|e| e.to_string())?;
|
||||||
|
|
||||||
// Extract frames to canonical zero-padded names; pixiv ugoira
|
// Extract frames to canonical zero-padded names; pixiv ugoira
|
||||||
// frames are uniformly jpg or png per artwork.
|
// frames are uniformly jpg or png per artwork.
|
||||||
let mut archive = zip::ZipArchive::new(Cursor::new(zip_bytes))
|
let mut archive = zip::ZipArchive::new(Cursor::new(zip_bytes))
|
||||||
.map_err(|e| format!("unzip: {e}"))?;
|
.map_err(|e| format!("unzip: {e}"))?;
|
||||||
// pixiv ugoira frames are uniformly jpg or png per artwork; take
|
// pixiv ugoira frames are uniformly jpg or png per artwork; take
|
||||||
// the extension from the first entry.
|
// the extension from the first entry.
|
||||||
let extension = if archive.len() > 0 {
|
let extension = if archive.len() > 0 {
|
||||||
let first_name = archive
|
let first_name = archive
|
||||||
.by_index(0)
|
.by_index(0)
|
||||||
.map_err(|e| e.to_string())?
|
.map_err(|e| e.to_string())?
|
||||||
.name()
|
.name()
|
||||||
.to_string();
|
.to_string();
|
||||||
first_name
|
first_name.rsplit('.').next().unwrap_or("jpg").to_string()
|
||||||
.rsplit('.')
|
} else {
|
||||||
.next()
|
"jpg".to_string()
|
||||||
.unwrap_or("jpg")
|
};
|
||||||
.to_string()
|
let mut count = 0usize;
|
||||||
} else {
|
for i in 0..archive.len() {
|
||||||
"jpg".to_string()
|
let mut entry = archive.by_index(i).map_err(|e| e.to_string())?;
|
||||||
};
|
let mut bytes = Vec::new();
|
||||||
let mut count = 0usize;
|
entry.read_to_end(&mut bytes).map_err(|e| e.to_string())?;
|
||||||
for i in 0..archive.len() {
|
let path = frames_dir
|
||||||
let mut entry = archive.by_index(i).map_err(|e| e.to_string())?;
|
.path()
|
||||||
let mut bytes = Vec::new();
|
.join(format!("img_{count:05}.{extension}"));
|
||||||
entry.read_to_end(&mut bytes).map_err(|e| e.to_string())?;
|
std::fs::write(&path, bytes).map_err(|e| e.to_string())?;
|
||||||
let path = frames_dir.path().join(format!("img_{count:05}.{extension}"));
|
count += 1;
|
||||||
std::fs::write(&path, bytes).map_err(|e| e.to_string())?;
|
}
|
||||||
count += 1;
|
if count == 0 {
|
||||||
}
|
return Err("empty frame zip".to_string());
|
||||||
if count == 0 {
|
}
|
||||||
return Err("empty frame zip".to_string());
|
|
||||||
}
|
|
||||||
|
|
||||||
// Constant rate from the median frame delay (ms).
|
// Constant rate from the median frame delay (ms).
|
||||||
let mut delays = frame_delays;
|
let mut delays = frame_delays;
|
||||||
delays.sort_unstable();
|
delays.sort_unstable();
|
||||||
let median = delays[delays.len() / 2].max(1);
|
let median = delays[delays.len() / 2].max(1);
|
||||||
let framerate = 1000.0 / median as f64;
|
let framerate = 1000.0 / median as f64;
|
||||||
|
|
||||||
let output = out_dir.path().join("ugoira.mp4");
|
let output = out_dir.path().join("ugoira.mp4");
|
||||||
let status = std::process::Command::new("ffmpeg")
|
let status = std::process::Command::new("ffmpeg")
|
||||||
.args([
|
.args([
|
||||||
"-y",
|
"-y",
|
||||||
"-framerate",
|
"-framerate",
|
||||||
&framerate.to_string(),
|
&framerate.to_string(),
|
||||||
"-i",
|
"-i",
|
||||||
&frames_dir.path().join(format!("img_%05d.{extension}")).to_string_lossy(),
|
&frames_dir
|
||||||
// libx264 needs even dimensions; pixiv ugoira frames can
|
.path()
|
||||||
// be odd-sized (e.g. 277x405).
|
.join(format!("img_%05d.{extension}"))
|
||||||
"-vf",
|
.to_string_lossy(),
|
||||||
"scale=trunc(iw/2)*2:trunc(ih/2)*2",
|
// libx264 needs even dimensions; pixiv ugoira frames can
|
||||||
"-c:v",
|
// be odd-sized (e.g. 277x405).
|
||||||
"libx264",
|
"-vf",
|
||||||
"-pix_fmt",
|
"scale=trunc(iw/2)*2:trunc(ih/2)*2",
|
||||||
"yuv420p",
|
"-c:v",
|
||||||
"-movflags",
|
"libx264",
|
||||||
"+faststart",
|
"-pix_fmt",
|
||||||
&output.to_string_lossy(),
|
"yuv420p",
|
||||||
])
|
"-movflags",
|
||||||
.stdout(std::process::Stdio::null())
|
"+faststart",
|
||||||
.stderr(std::process::Stdio::null())
|
&output.to_string_lossy(),
|
||||||
.status()
|
])
|
||||||
.map_err(|e| format!("ffmpeg spawn failed: {e}"))?;
|
.stdout(std::process::Stdio::null())
|
||||||
if !status.success() {
|
.stderr(std::process::Stdio::null())
|
||||||
return Err(format!("ffmpeg exited with {status}"));
|
.status()
|
||||||
}
|
.map_err(|e| format!("ffmpeg spawn failed: {e}"))?;
|
||||||
Ok((output.to_string_lossy().into_owned(), out_dir))
|
if !status.success() {
|
||||||
},
|
return Err(format!("ffmpeg exited with {status}"));
|
||||||
)
|
}
|
||||||
.await
|
Ok((output.to_string_lossy().into_owned(), out_dir))
|
||||||
.expect("ugoira encode worker panicked");
|
})
|
||||||
|
.await
|
||||||
|
.expect("ugoira encode worker panicked");
|
||||||
match result {
|
match result {
|
||||||
Ok(pair) => Ok(Some(pair)),
|
Ok(pair) => Ok(Some(pair)),
|
||||||
Err(message) => {
|
Err(message) => {
|
||||||
@@ -332,9 +338,8 @@ fn log_once_ffmpeg_missing() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// pixiv3-rs replacement: `None` when `PIXIV_REFRESH_TOKEN` is unset.
|
/// pixiv3-rs replacement: `None` when `PIXIV_REFRESH_TOKEN` is unset.
|
||||||
static PIXIV_CLIENT: LazyLock<Option<PixivAPI>> = LazyLock::new(|| {
|
static PIXIV_CLIENT: LazyLock<Option<PixivAPI>> =
|
||||||
env::var("PIXIV_REFRESH_TOKEN").ok().map(PixivAPI::new)
|
LazyLock::new(|| env::var("PIXIV_REFRESH_TOKEN").ok().map(PixivAPI::new));
|
||||||
});
|
|
||||||
|
|
||||||
/// Set at startup when the login validation fails; pixiv stays disabled until
|
/// Set at startup when the login validation fails; pixiv stays disabled until
|
||||||
/// the next process start.
|
/// the next process start.
|
||||||
|
|||||||
@@ -82,12 +82,15 @@ impl Illustration {
|
|||||||
// keeps media empty when encoding fails or ffmpeg is missing.
|
// keeps media empty when encoding fails or ffmpeg is missing.
|
||||||
} else if model.page_count > 1 {
|
} else if model.page_count > 1 {
|
||||||
media.extend(model.meta_pages.iter().filter_map(|page| {
|
media.extend(model.meta_pages.iter().filter_map(|page| {
|
||||||
page.image_urls.original.clone().map(|original| Media::Illustration {
|
page.image_urls
|
||||||
title: None,
|
.original
|
||||||
url: original,
|
.clone()
|
||||||
thumbnail_url: Some(page.image_urls.medium.clone()),
|
.map(|original| Media::Illustration {
|
||||||
fallback_url: Some(page.image_urls.large.clone()),
|
title: None,
|
||||||
})
|
url: original,
|
||||||
|
thumbnail_url: Some(page.image_urls.medium.clone()),
|
||||||
|
fallback_url: Some(page.image_urls.large.clone()),
|
||||||
|
})
|
||||||
}));
|
}));
|
||||||
} else if let Some(original) = model
|
} else if let Some(original) = model
|
||||||
.meta_single_page
|
.meta_single_page
|
||||||
@@ -147,8 +150,8 @@ impl From<Illustration> for Fetched {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
|
||||||
use super::super::model::IllustrationModel;
|
use super::super::model::IllustrationModel;
|
||||||
|
use super::*;
|
||||||
|
|
||||||
fn illust_json(
|
fn illust_json(
|
||||||
type_: &str,
|
type_: &str,
|
||||||
@@ -203,8 +206,14 @@ mod tests {
|
|||||||
("https://pixiv.net/artworks/123456", "123456"),
|
("https://pixiv.net/artworks/123456", "123456"),
|
||||||
("https://www.pixiv.net/en/artworks/123456", "123456"),
|
("https://www.pixiv.net/en/artworks/123456", "123456"),
|
||||||
("https://www.pixiv.net/i/123456", "123456"),
|
("https://www.pixiv.net/i/123456", "123456"),
|
||||||
("https://www.pixiv.net/member_illust.php?mode=medium&illust_id=123456", "123456"),
|
(
|
||||||
("https://www.pixiv.net/en/member_illust.php?illust_id=123456", "123456"),
|
"https://www.pixiv.net/member_illust.php?mode=medium&illust_id=123456",
|
||||||
|
"123456",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"https://www.pixiv.net/en/member_illust.php?illust_id=123456",
|
||||||
|
"123456",
|
||||||
|
),
|
||||||
];
|
];
|
||||||
for (url, id) in cases {
|
for (url, id) in cases {
|
||||||
let caps = PATTERN.captures(url).unwrap_or_else(|| panic!("{url}"));
|
let caps = PATTERN.captures(url).unwrap_or_else(|| panic!("{url}"));
|
||||||
@@ -225,7 +234,14 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn ugoira_yields_empty_media() {
|
fn ugoira_yields_empty_media() {
|
||||||
let v = illust_json("ugoira", 1, Some("https://i.pximg.net/orig.jpg"), None, vec![], 0);
|
let v = illust_json(
|
||||||
|
"ugoira",
|
||||||
|
1,
|
||||||
|
Some("https://i.pximg.net/orig.jpg"),
|
||||||
|
None,
|
||||||
|
vec![],
|
||||||
|
0,
|
||||||
|
);
|
||||||
let illustration = parse(v);
|
let illustration = parse(v);
|
||||||
let fetched: Fetched = illustration.into();
|
let fetched: Fetched = illustration.into();
|
||||||
assert!(fetched.media.is_empty());
|
assert!(fetched.media.is_empty());
|
||||||
@@ -296,7 +312,12 @@ mod tests {
|
|||||||
let fetched: Fetched = parse(v).into();
|
let fetched: Fetched = parse(v).into();
|
||||||
assert_eq!(fetched.media.len(), 1);
|
assert_eq!(fetched.media.len(), 1);
|
||||||
match &fetched.media[0] {
|
match &fetched.media[0] {
|
||||||
Media::Illustration { url, thumbnail_url, fallback_url, .. } => {
|
Media::Illustration {
|
||||||
|
url,
|
||||||
|
thumbnail_url,
|
||||||
|
fallback_url,
|
||||||
|
..
|
||||||
|
} => {
|
||||||
assert_eq!(url, "https://i.pximg.net/p2.jpg");
|
assert_eq!(url, "https://i.pximg.net/p2.jpg");
|
||||||
assert_eq!(thumbnail_url.as_deref(), Some("m2.jpg"));
|
assert_eq!(thumbnail_url.as_deref(), Some("m2.jpg"));
|
||||||
assert_eq!(fallback_url.as_deref(), Some("l2.jpg"));
|
assert_eq!(fallback_url.as_deref(), Some("l2.jpg"));
|
||||||
@@ -307,7 +328,14 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn caption_with_escapes_format_and_substitutes() {
|
fn caption_with_escapes_format_and_substitutes() {
|
||||||
let v = illust_json("illust", 1, Some("https://i.pximg.net/o.jpg"), None, vec![], 0);
|
let v = illust_json(
|
||||||
|
"illust",
|
||||||
|
1,
|
||||||
|
Some("https://i.pximg.net/o.jpg"),
|
||||||
|
None,
|
||||||
|
vec![],
|
||||||
|
0,
|
||||||
|
);
|
||||||
let fetched: Fetched = parse(v).into();
|
let fetched: Fetched = parse(v).into();
|
||||||
// Format string is escaped in full, then placeholders substituted.
|
// Format string is escaped in full, then placeholders substituted.
|
||||||
let out = fetched.caption_with("{title} by {author} <script> {tags}");
|
let out = fetched.caption_with("{title} by {author} <script> {tags}");
|
||||||
@@ -330,7 +358,14 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn ai_work_gets_leading_ai_tag() {
|
fn ai_work_gets_leading_ai_tag() {
|
||||||
// illust_ai_type == 2 is the only AI marker.
|
// illust_ai_type == 2 is the only AI marker.
|
||||||
let v = illust_json("illust", 1, Some("https://i.pximg.net/o.jpg"), None, vec![], 2);
|
let v = illust_json(
|
||||||
|
"illust",
|
||||||
|
1,
|
||||||
|
Some("https://i.pximg.net/o.jpg"),
|
||||||
|
None,
|
||||||
|
vec![],
|
||||||
|
2,
|
||||||
|
);
|
||||||
let fetched: Fetched = parse(v).into();
|
let fetched: Fetched = parse(v).into();
|
||||||
assert!(
|
assert!(
|
||||||
fetched.caption.contains("#AI #tag1 #tag2"),
|
fetched.caption.contains("#AI #tag1 #tag2"),
|
||||||
@@ -338,14 +373,25 @@ mod tests {
|
|||||||
fetched.caption
|
fetched.caption
|
||||||
);
|
);
|
||||||
// The {tags} placeholder reflects the tag array too.
|
// The {tags} placeholder reflects the tag array too.
|
||||||
assert!(fetched.caption_with("{tags}").starts_with("#AI "), "got: {}", fetched.caption_with("{tags}"));
|
assert!(
|
||||||
|
fetched.caption_with("{tags}").starts_with("#AI "),
|
||||||
|
"got: {}",
|
||||||
|
fetched.caption_with("{tags}")
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn non_ai_work_has_no_ai_tag() {
|
fn non_ai_work_has_no_ai_tag() {
|
||||||
// 1 = explicitly not AI, 0 = undefined: neither gets the #AI tag.
|
// 1 = explicitly not AI, 0 = undefined: neither gets the #AI tag.
|
||||||
for ai_type in [0, 1] {
|
for ai_type in [0, 1] {
|
||||||
let v = illust_json("illust", 1, Some("https://i.pximg.net/o.jpg"), None, vec![], ai_type);
|
let v = illust_json(
|
||||||
|
"illust",
|
||||||
|
1,
|
||||||
|
Some("https://i.pximg.net/o.jpg"),
|
||||||
|
None,
|
||||||
|
vec![],
|
||||||
|
ai_type,
|
||||||
|
);
|
||||||
let fetched: Fetched = parse(v).into();
|
let fetched: Fetched = parse(v).into();
|
||||||
assert!(
|
assert!(
|
||||||
!fetched.caption.contains("#AI"),
|
!fetched.caption.contains("#AI"),
|
||||||
@@ -357,7 +403,14 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn caption_escapes_and_links() {
|
fn caption_escapes_and_links() {
|
||||||
let v = illust_json("illust", 1, Some("https://i.pximg.net/o.jpg"), None, vec![], 0);
|
let v = illust_json(
|
||||||
|
"illust",
|
||||||
|
1,
|
||||||
|
Some("https://i.pximg.net/o.jpg"),
|
||||||
|
None,
|
||||||
|
vec![],
|
||||||
|
0,
|
||||||
|
);
|
||||||
let fetched: Fetched = parse(v).into();
|
let fetched: Fetched = parse(v).into();
|
||||||
assert!(
|
assert!(
|
||||||
fetched
|
fetched
|
||||||
@@ -367,9 +420,6 @@ mod tests {
|
|||||||
fetched.caption
|
fetched.caption
|
||||||
);
|
);
|
||||||
assert!(fetched.caption.contains("#tag1 #tag2"));
|
assert!(fetched.caption.contains("#tag1 #tag2"));
|
||||||
assert_eq!(
|
assert_eq!(fetched.source_url, "https://www.pixiv.net/artworks/123");
|
||||||
fetched.source_url,
|
|
||||||
"https://www.pixiv.net/artworks/123"
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,4 +3,4 @@ mod interface;
|
|||||||
mod model;
|
mod model;
|
||||||
|
|
||||||
pub use api::{PixivAPI, PixivError, disable, fetch, validate};
|
pub use api::{PixivAPI, PixivError, disable, fetch, validate};
|
||||||
pub use interface::{PATTERN, Illustration, enabled, fetch_from_url};
|
pub use interface::{Illustration, PATTERN, enabled, fetch_from_url};
|
||||||
|
|||||||
@@ -23,7 +23,7 @@
|
|||||||
|
|
||||||
use std::sync::LazyLock;
|
use std::sync::LazyLock;
|
||||||
|
|
||||||
use serde_json::{json, Value};
|
use serde_json::{Value, json};
|
||||||
|
|
||||||
use crate::site::FetchError;
|
use crate::site::FetchError;
|
||||||
|
|
||||||
@@ -40,8 +40,7 @@ static AUTH_TOKEN: LazyLock<Option<String>> = LazyLock::new(|| {
|
|||||||
});
|
});
|
||||||
|
|
||||||
/// Public "logged in" client token used by the x.com web app.
|
/// Public "logged in" client token used by the x.com web app.
|
||||||
const LOGGED_IN_BEARER: &str =
|
const LOGGED_IN_BEARER: &str = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA";
|
||||||
"Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA";
|
|
||||||
|
|
||||||
/// `TweetDetail` query id (from nazurin; still valid as of 2026-08,
|
/// `TweetDetail` query id (from nazurin; still valid as of 2026-08,
|
||||||
/// corroborated by the current FxEmbed build — see module caveats).
|
/// corroborated by the current FxEmbed build — see module caveats).
|
||||||
@@ -103,9 +102,7 @@ pub fn enabled() -> bool {
|
|||||||
/// Fetches a tweet as the logged-in user via the private GraphQL API.
|
/// Fetches a tweet as the logged-in user via the private GraphQL API.
|
||||||
/// Returns the syndication-shaped [`Tweet`] (media included for NSFW posts).
|
/// Returns the syndication-shaped [`Tweet`] (media included for NSFW posts).
|
||||||
pub async fn fetch(id: &str) -> Result<Tweet, FetchError> {
|
pub async fn fetch(id: &str) -> Result<Tweet, FetchError> {
|
||||||
let token = AUTH_TOKEN
|
let token = AUTH_TOKEN.as_deref().ok_or(FetchError::Sensitive)?;
|
||||||
.as_deref()
|
|
||||||
.ok_or(FetchError::Sensitive)?;
|
|
||||||
// 16 random bytes as 32 hex chars: X rejects ct0 values of any other
|
// 16 random bytes as 32 hex chars: X rejects ct0 values of any other
|
||||||
// length with 403 code 353 ("matching csrf cookie and header").
|
// length with 403 code 353 ("matching csrf cookie and header").
|
||||||
let ct0: String = (0..16)
|
let ct0: String = (0..16)
|
||||||
@@ -136,11 +133,12 @@ pub async fn fetch(id: &str) -> Result<Tweet, FetchError> {
|
|||||||
let text = response.text().await?;
|
let text = response.text().await?;
|
||||||
let json: Value = serde_json::from_str(&text)?;
|
let json: Value = serde_json::from_str(&text)?;
|
||||||
let result = parse_tweet_result(&json, id)?;
|
let result = parse_tweet_result(&json, id)?;
|
||||||
let syndication_shape = to_syndication_shape(&result)
|
let syndication_shape = to_syndication_shape(&result).ok_or_else(|| {
|
||||||
.ok_or_else(|| FetchError::Json(serde_json::Error::io(std::io::Error::new(
|
FetchError::Json(serde_json::Error::io(std::io::Error::new(
|
||||||
std::io::ErrorKind::InvalidData,
|
std::io::ErrorKind::InvalidData,
|
||||||
"missing tweet fields in GraphQL response",
|
"missing tweet fields in GraphQL response",
|
||||||
))))?;
|
)))
|
||||||
|
})?;
|
||||||
Tweet::from_syndication_json(&syndication_shape.to_string()).map_err(FetchError::Json)
|
Tweet::from_syndication_json(&syndication_shape.to_string()).map_err(FetchError::Json)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -228,7 +226,6 @@ fn to_syndication_shape(tweet: &Value) -> Option<Value> {
|
|||||||
"screen_name": user.get("screen_name"),
|
"screen_name": user.get("screen_name"),
|
||||||
},
|
},
|
||||||
"possibly_sensitive": legacy.get("possibly_sensitive"),
|
"possibly_sensitive": legacy.get("possibly_sensitive"),
|
||||||
"display_text_range": legacy.get("display_text_range"),
|
|
||||||
"entities": legacy.get("entities"),
|
"entities": legacy.get("entities"),
|
||||||
"mediaDetails": legacy.pointer("/extended_entities/media"),
|
"mediaDetails": legacy.pointer("/extended_entities/media"),
|
||||||
}))
|
}))
|
||||||
@@ -251,12 +248,12 @@ mod tests {
|
|||||||
"legacy": {
|
"legacy": {
|
||||||
"id_str": "2083868672721039569",
|
"id_str": "2083868672721039569",
|
||||||
"full_text": "nsfw content https://t.co/abc123",
|
"full_text": "nsfw content https://t.co/abc123",
|
||||||
"display_text_range": [0, 12],
|
|
||||||
"possibly_sensitive": true,
|
"possibly_sensitive": true,
|
||||||
"entities": {
|
"entities": {
|
||||||
"urls": [
|
// The appended media link lives in extended_entities.media,
|
||||||
{ "url": "https://t.co/abc123", "expanded_url": "https://example.com/x" }
|
// not entities.urls, so it has no expansion mapping and the
|
||||||
]
|
// content-based strip removes it.
|
||||||
|
"urls": []
|
||||||
},
|
},
|
||||||
"extended_entities": {
|
"extended_entities": {
|
||||||
"media": [
|
"media": [
|
||||||
@@ -318,8 +315,11 @@ mod tests {
|
|||||||
}
|
}
|
||||||
other => panic!("expected video, got {other:?}"),
|
other => panic!("expected video, got {other:?}"),
|
||||||
}
|
}
|
||||||
assert_eq!(fetched.source_url, "https://x.com/nsfw_author/status/2083868672721039569");
|
assert_eq!(
|
||||||
// display_text_range cuts the trailing t.co link.
|
fetched.source_url,
|
||||||
|
"https://x.com/nsfw_author/status/2083868672721039569"
|
||||||
|
);
|
||||||
|
// The appended media short link (no URL-entity mapping) is stripped.
|
||||||
assert_eq!(fetched.title, "nsfw content");
|
assert_eq!(fetched.title, "nsfw content");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -331,7 +331,10 @@ mod tests {
|
|||||||
let json = conversation(rt);
|
let json = conversation(rt);
|
||||||
let result = parse_tweet_result(&json, "2083868672721039569").unwrap();
|
let result = parse_tweet_result(&json, "2083868672721039569").unwrap();
|
||||||
assert!(result.pointer("/legacy/retweeted_status_result").is_none());
|
assert!(result.pointer("/legacy/retweeted_status_result").is_none());
|
||||||
assert_eq!(result.pointer("/legacy/id_str").unwrap(), "2083868672721039569");
|
assert_eq!(
|
||||||
|
result.pointer("/legacy/id_str").unwrap(),
|
||||||
|
"2083868672721039569"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -35,9 +35,7 @@ pub async fn fetch_from_url(url: &str) -> Result<Fetched, FetchError> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
log::info!(
|
log::info!("tweet {id} is sensitive; set TWITTER_AUTH_TOKEN to fetch NSFW media");
|
||||||
"tweet {id} is sensitive; set TWITTER_AUTH_TOKEN to fetch NSFW media"
|
|
||||||
);
|
|
||||||
Ok(empty_fetched(url))
|
Ok(empty_fetched(url))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -158,12 +156,10 @@ impl Tweet {
|
|||||||
pub fn from_syndication_json(raw_json: &str) -> Result<Self, serde_json::Error> {
|
pub fn from_syndication_json(raw_json: &str) -> Result<Self, serde_json::Error> {
|
||||||
let json: model::SyndicationTweet = serde_json::from_str(raw_json)?;
|
let json: model::SyndicationTweet = serde_json::from_str(raw_json)?;
|
||||||
let id = json.id_str;
|
let id = json.id_str;
|
||||||
// Strip the appended media short link first, then expand the remaining
|
// Expand the user's t.co short links to their real destinations and
|
||||||
// t.co short links (the user's own URLs) to their real destinations.
|
// strip the appended media short link, mirroring FxEmbed's linkFixer
|
||||||
let text = expand_links(
|
// (no display_text_range arithmetic — see expand_links).
|
||||||
&strip_trailing_short_links(&json.text, json.display_text_range),
|
let text = expand_links(&json.text, &json.entities.urls);
|
||||||
&json.entities.urls,
|
|
||||||
);
|
|
||||||
// `name` is the display name, `screen_name` the handle (Python's
|
// `name` is the display name, `screen_name` the handle (Python's
|
||||||
// vxtwitter mapping: author = display name, author_id = handle).
|
// vxtwitter mapping: author = display name, author_id = handle).
|
||||||
let author = json.user.name;
|
let author = json.user.name;
|
||||||
@@ -204,51 +200,48 @@ impl Tweet {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The raw syndication `text` ends with the appended media short link
|
/// Mirrors FxEmbed's `linkFixer` (link-fixer.ts): expand every t.co short
|
||||||
/// (" https://t.co/wmI8McgXul"). `display_text_range` marks the visible text;
|
/// link that has an entity mapping to its real destination, drop internal
|
||||||
/// a regex strips any remaining trailing t.co link when the range is absent
|
/// `x.com/i/web/status/…` plumbing links, then strip any remaining t.co
|
||||||
/// or a tweet ends in a URL short link.
|
/// short link (the appended media link and other unmapped short links).
|
||||||
///
|
/// Pure content matching — no `display_text_range` arithmetic, so the
|
||||||
/// X reports these indices in Unicode **code points**, not UTF-16 units
|
/// endpoint's inconsistent index units (UTF-16 vs code points, see the
|
||||||
/// (verified against GraphQL responses containing emoji: cutting an emoji
|
/// deleted `strip_trailing_short_links`) never matter.
|
||||||
/// tweet by UTF-16 units silently drops the character after the emoji).
|
|
||||||
fn strip_trailing_short_links(text: &str, display_text_range: Option<[usize; 2]>) -> String {
|
|
||||||
let mut out = match display_text_range {
|
|
||||||
Some([start, end]) if start < end => {
|
|
||||||
text.chars().skip(start).take(end - start).collect()
|
|
||||||
}
|
|
||||||
_ => text.to_string(),
|
|
||||||
};
|
|
||||||
while TRAILING_TCO.is_match(&out) {
|
|
||||||
out = TRAILING_TCO.replace(&out, "").into_owned();
|
|
||||||
}
|
|
||||||
out
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Trailing Twitter short link, optionally preceded by whitespace.
|
|
||||||
static TRAILING_TCO: LazyLock<Regex> = LazyLock::new(|| {
|
|
||||||
Regex::new(r"\s*https?://t\.co/[A-Za-z0-9]+$").unwrap()
|
|
||||||
});
|
|
||||||
|
|
||||||
/// Replaces every t.co short link that has an entity mapping with its
|
|
||||||
/// expanded URL. Short links without a mapping stay untouched.
|
|
||||||
fn expand_links(text: &str, urls: &[model::SyndicationEntityUrl]) -> String {
|
fn expand_links(text: &str, urls: &[model::SyndicationEntityUrl]) -> String {
|
||||||
let mut out = text.to_string();
|
let mut out = text.to_string();
|
||||||
for entity in urls {
|
for entity in urls {
|
||||||
if let Some(expanded) = &entity.expanded_url {
|
let Some(expanded) = &entity.expanded_url else {
|
||||||
out = out.replace(&entity.url, expanded);
|
continue;
|
||||||
}
|
};
|
||||||
|
let replacement = if WEB_STATUS_URL.is_match(expanded) {
|
||||||
|
""
|
||||||
|
} else {
|
||||||
|
expanded
|
||||||
|
};
|
||||||
|
out = out.replace(&entity.url, replacement);
|
||||||
}
|
}
|
||||||
out
|
TCO_LINK.replace_all(&out, "").into_owned()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Internal x.com page links (reply / quote plumbing) expand to
|
||||||
|
/// `x.com/i/web/status/<id>`; FxEmbed drops them — the tweet's own content
|
||||||
|
/// already carries the information.
|
||||||
|
static WEB_STATUS_URL: LazyLock<Regex> =
|
||||||
|
LazyLock::new(|| Regex::new(r"^https://(?:x\.com|twitter\.com)/i/web/status/\w+").unwrap());
|
||||||
|
|
||||||
|
/// A t.co short link, optionally preceded by a space. Any leftover
|
||||||
|
/// occurrence (unmapped — e.g. the appended media link) is removed,
|
||||||
|
/// mirroring FxEmbed. Real short-link codes are 10 alphanumerics; the
|
||||||
|
/// length-agnostic class keeps fixtures and hypothetical odd lengths safe.
|
||||||
|
static TCO_LINK: LazyLock<Regex> =
|
||||||
|
LazyLock::new(|| Regex::new(r" ?https?://t\.co/[A-Za-z0-9]+").unwrap());
|
||||||
|
|
||||||
/// pbs.twimg.com serves a reduced default size without size params; `name=orig`
|
/// pbs.twimg.com serves a reduced default size without size params; `name=orig`
|
||||||
/// returns the original file (fxtwitter used to hand out the original
|
/// returns the original file (fxtwitter used to hand out the original
|
||||||
/// directly, the syndication API does not). Non-twimg URLs pass through
|
/// directly, the syndication API does not). Non-twimg URLs pass through
|
||||||
/// unchanged.
|
/// unchanged.
|
||||||
fn original_twimg_url(url: &str) -> String {
|
fn original_twimg_url(url: &str) -> String {
|
||||||
if url.starts_with("https://pbs.twimg.com/")
|
if url.starts_with("https://pbs.twimg.com/") && (url.ends_with(".jpg") || url.ends_with(".png"))
|
||||||
&& (url.ends_with(".jpg") || url.ends_with(".png"))
|
|
||||||
{
|
{
|
||||||
format!("{url}?name=orig")
|
format!("{url}?name=orig")
|
||||||
} else {
|
} else {
|
||||||
@@ -363,24 +356,23 @@ mod tests {
|
|||||||
match &fetched.media[0] {
|
match &fetched.media[0] {
|
||||||
Media::Illustration { url, .. } => {
|
Media::Illustration { url, .. } => {
|
||||||
// Photo URL is rewritten to request the original file.
|
// Photo URL is rewritten to request the original file.
|
||||||
assert_eq!(
|
assert_eq!(url, "https://pbs.twimg.com/media/photo.jpg?name=orig");
|
||||||
url,
|
|
||||||
"https://pbs.twimg.com/media/photo.jpg?name=orig"
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
other => panic!("expected illustration, got {other:?}"),
|
other => panic!("expected illustration, got {other:?}"),
|
||||||
}
|
}
|
||||||
match &fetched.media[1] {
|
match &fetched.media[1] {
|
||||||
Media::Video { url, thumbnail_url, .. } => {
|
Media::Video {
|
||||||
|
url, thumbnail_url, ..
|
||||||
|
} => {
|
||||||
assert_eq!(url, "https://video.twimg.com/v.mp4");
|
assert_eq!(url, "https://video.twimg.com/v.mp4");
|
||||||
assert_eq!(thumbnail_url, "https://pbs.twimg.com/thumb.jpg");
|
assert_eq!(thumbnail_url, "https://pbs.twimg.com/thumb.jpg");
|
||||||
}
|
}
|
||||||
other => panic!("expected video, got {other:?}"),
|
other => panic!("expected video, got {other:?}"),
|
||||||
}
|
}
|
||||||
assert!(
|
assert!(
|
||||||
fetched
|
fetched.caption.contains(
|
||||||
.caption
|
"<a href=\"https://x.com/author_handle\">Display Name</a>: a & b <c>"
|
||||||
.contains("<a href=\"https://x.com/author_handle\">Display Name</a>: a & b <c>"),
|
),
|
||||||
"caption: {}",
|
"caption: {}",
|
||||||
fetched.caption
|
fetched.caption
|
||||||
);
|
);
|
||||||
@@ -411,13 +403,12 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn syndication_text_strips_trailing_media_short_link() {
|
fn syndication_text_strips_trailing_media_short_link() {
|
||||||
// Real syndication shape: the media short link sits after the visible
|
// Real syndication shape: the appended media short link sits after the
|
||||||
// text, and display_text_range marks where it begins.
|
// visible text; the unmapped t.co link is stripped by content.
|
||||||
let raw = serde_json::json!({
|
let raw = serde_json::json!({
|
||||||
"__typename": "Tweet",
|
"__typename": "Tweet",
|
||||||
"id_str": "1",
|
"id_str": "1",
|
||||||
"text": "hello world https://t.co/abc123",
|
"text": "hello world https://t.co/abc123",
|
||||||
"display_text_range": [0, 11],
|
|
||||||
"user": { "name": "N", "screen_name": "h" },
|
"user": { "name": "N", "screen_name": "h" },
|
||||||
"mediaDetails": []
|
"mediaDetails": []
|
||||||
});
|
});
|
||||||
@@ -427,8 +418,31 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn syndication_text_strips_trailing_short_link_without_range() {
|
fn syndication_text_strips_trailing_link_regardless_of_index_units() {
|
||||||
// No display_text_range: the regex fallback removes the trailing link.
|
// Real tweet 2084567054481571919: the visible text is 30 code points
|
||||||
|
// but 41 UTF-16 units, and the two endpoints historically reported
|
||||||
|
// display_text_range in different units (UTF-16 on syndication, code
|
||||||
|
// points on GraphQL). The FxEmbed-style content-based strip ignores
|
||||||
|
// the range entirely, so the appended media link is removed for any
|
||||||
|
// response shape.
|
||||||
|
let text = "妄想𝑨𝒅𝒅𝒊𝒄𝒕𝒊𝒐𝒏…🩷💚❤️\n#ゼンゼロ #zzzero https://t.co/XnIi83EkEB";
|
||||||
|
let visible = "妄想𝑨𝒅𝒅𝒊𝒄𝒕𝒊𝒐𝒏…🩷💚❤️\n#ゼンゼロ #zzzero";
|
||||||
|
let raw = serde_json::json!({
|
||||||
|
"__typename": "Tweet",
|
||||||
|
"id_str": "2084567054481571919",
|
||||||
|
"text": text,
|
||||||
|
"user": { "name": "N", "screen_name": "h" },
|
||||||
|
"mediaDetails": []
|
||||||
|
});
|
||||||
|
let tweet = Tweet::from_syndication_json(&raw.to_string()).unwrap();
|
||||||
|
assert_eq!(tweet.text, visible, "left a partial link");
|
||||||
|
assert!(!tweet.caption().contains("t.co"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn syndication_text_strips_trailing_short_link_without_entities() {
|
||||||
|
// No URL entities at all: the leftover t.co link is stripped by the
|
||||||
|
// content regex.
|
||||||
let raw = serde_json::json!({
|
let raw = serde_json::json!({
|
||||||
"__typename": "Tweet",
|
"__typename": "Tweet",
|
||||||
"id_str": "1",
|
"id_str": "1",
|
||||||
@@ -449,7 +463,6 @@ mod tests {
|
|||||||
"__typename": "Tweet",
|
"__typename": "Tweet",
|
||||||
"id_str": "1",
|
"id_str": "1",
|
||||||
"text": "Test Tweet with @mentionThis $twtr https://t.co/RzmrQ6wAzD #hashtag https://t.co/9r69akA484",
|
"text": "Test Tweet with @mentionThis $twtr https://t.co/RzmrQ6wAzD #hashtag https://t.co/9r69akA484",
|
||||||
"display_text_range": [0, 67],
|
|
||||||
"user": { "name": "N", "screen_name": "h" },
|
"user": { "name": "N", "screen_name": "h" },
|
||||||
"entities": {
|
"entities": {
|
||||||
"urls": [{
|
"urls": [{
|
||||||
@@ -469,25 +482,47 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn syndication_text_keeps_unmapped_short_links() {
|
fn syndication_text_strips_unmapped_short_links() {
|
||||||
// No entity mapping for the embedded link: it stays as-is. Only the
|
// FxEmbed parity: short links without an entity mapping (appended
|
||||||
// trailing media link is stripped.
|
// media link, embedded unmapped links) are stripped, not kept.
|
||||||
let raw = serde_json::json!({
|
let raw = serde_json::json!({
|
||||||
"__typename": "Tweet",
|
"__typename": "Tweet",
|
||||||
"id_str": "1",
|
"id_str": "1",
|
||||||
"text": "check https://t.co/abc123 #tag https://t.co/def456",
|
"text": "check https://t.co/abc123 #tag https://t.co/def456",
|
||||||
"display_text_range": [0, 30],
|
|
||||||
"user": { "name": "N", "screen_name": "h" },
|
"user": { "name": "N", "screen_name": "h" },
|
||||||
"mediaDetails": []
|
"mediaDetails": []
|
||||||
});
|
});
|
||||||
let tweet = Tweet::from_syndication_json(&raw.to_string()).unwrap();
|
let tweet = Tweet::from_syndication_json(&raw.to_string()).unwrap();
|
||||||
assert_eq!(tweet.text, "check https://t.co/abc123 #tag");
|
assert_eq!(tweet.text, "check #tag");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn syndication_text_utf16_display_range_keeps_multibyte() {
|
fn syndication_text_drops_internal_web_status_links() {
|
||||||
// display_text_range is in UTF-16 units; a Japanese text must not be
|
// FxEmbed parity: a mapped link expanding to an internal
|
||||||
// sliced by UTF-8 bytes.
|
// x.com/i/web/status/... page (reply/quote plumbing) is removed
|
||||||
|
// instead of being shown.
|
||||||
|
let raw = serde_json::json!({
|
||||||
|
"__typename": "Tweet",
|
||||||
|
"id_str": "1",
|
||||||
|
"text": "see https://t.co/xyz1234567 for context",
|
||||||
|
"user": { "name": "N", "screen_name": "h" },
|
||||||
|
"entities": {
|
||||||
|
"urls": [{
|
||||||
|
"url": "https://t.co/xyz1234567",
|
||||||
|
"expanded_url": "https://x.com/i/web/status/9876543210",
|
||||||
|
"display_url": "x.com/i/web/status/9876543210"
|
||||||
|
}]
|
||||||
|
},
|
||||||
|
"mediaDetails": []
|
||||||
|
});
|
||||||
|
let tweet = Tweet::from_syndication_json(&raw.to_string()).unwrap();
|
||||||
|
assert_eq!(tweet.text, "see for context");
|
||||||
|
assert!(!tweet.caption().contains("t.co"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn syndication_text_keeps_multibyte_text() {
|
||||||
|
// Text-only tweet: no short links, the multibyte text is untouched.
|
||||||
let text = "コミティア落ちたので、明日は行きません。🙏ごめんなさい";
|
let text = "コミティア落ちたので、明日は行きません。🙏ごめんなさい";
|
||||||
let units: Vec<u16> = text.encode_utf16().collect();
|
let units: Vec<u16> = text.encode_utf16().collect();
|
||||||
assert_eq!(units.len(), 28);
|
assert_eq!(units.len(), 28);
|
||||||
@@ -495,7 +530,6 @@ mod tests {
|
|||||||
"__typename": "Tweet",
|
"__typename": "Tweet",
|
||||||
"id_str": "1",
|
"id_str": "1",
|
||||||
"text": text,
|
"text": text,
|
||||||
"display_text_range": [0, 28],
|
|
||||||
"user": { "name": "N", "screen_name": "h" },
|
"user": { "name": "N", "screen_name": "h" },
|
||||||
"mediaDetails": []
|
"mediaDetails": []
|
||||||
});
|
});
|
||||||
@@ -547,6 +581,9 @@ mod tests {
|
|||||||
async fn live_fetch_deleted_tweet_is_not_found() {
|
async fn live_fetch_deleted_tweet_is_not_found() {
|
||||||
// Deleted tweet: the syndication endpoint answers with errors.
|
// Deleted tweet: the syndication endpoint answers with errors.
|
||||||
let result = fetch("0").await;
|
let result = fetch("0").await;
|
||||||
assert!(matches!(result, Err(FetchError::NotFound)), "got {result:?}");
|
assert!(
|
||||||
|
matches!(result, Err(FetchError::NotFound)),
|
||||||
|
"got {result:?}"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,10 +9,6 @@ pub struct SyndicationTweet {
|
|||||||
pub user: SyndicationUser,
|
pub user: SyndicationUser,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub possibly_sensitive: Option<bool>,
|
pub possibly_sensitive: Option<bool>,
|
||||||
/// Visible-text span; the raw `text` field has the appended media short
|
|
||||||
/// link after it. Indices are Unicode code points (not UTF-16 units).
|
|
||||||
#[serde(default, rename = "display_text_range")]
|
|
||||||
pub display_text_range: Option<[usize; 2]>,
|
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub entities: SyndicationEntities,
|
pub entities: SyndicationEntities,
|
||||||
#[serde(default, rename = "mediaDetails")]
|
#[serde(default, rename = "mediaDetails")]
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "xmedia-bot"
|
name = "xmedia-bot"
|
||||||
version = "1.0.7"
|
version = "1.0.8"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
|
|||||||
@@ -24,39 +24,37 @@ pub struct Config {
|
|||||||
impl Config {
|
impl Config {
|
||||||
pub fn load() -> Config {
|
pub fn load() -> Config {
|
||||||
let admin_ids = env::var("BOT_ADMIN")
|
let admin_ids = env::var("BOT_ADMIN")
|
||||||
.ok()
|
.ok()
|
||||||
.map(|s| {
|
.map(|s| {
|
||||||
s.split(',')
|
s.split(',')
|
||||||
.filter_map(|part| part.trim().parse::<i64>().ok())
|
.filter_map(|part| part.trim().parse::<i64>().ok())
|
||||||
.collect()
|
.collect()
|
||||||
})
|
})
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
|
||||||
let edit_message_ttl = env::var("EDIT_MESSAGE_TTL_SECONDS")
|
let edit_message_ttl = env::var("EDIT_MESSAGE_TTL_SECONDS")
|
||||||
.ok()
|
.ok()
|
||||||
.and_then(|s| s.parse::<u64>().ok())
|
.and_then(|s| s.parse::<u64>().ok())
|
||||||
.map(Duration::from_secs)
|
.map(Duration::from_secs)
|
||||||
.unwrap_or(Duration::from_secs(86400));
|
.unwrap_or(Duration::from_secs(86400));
|
||||||
|
|
||||||
let link_cache_ttl = env::var("LINK_CACHE_TTL_SECONDS")
|
let link_cache_ttl = env::var("LINK_CACHE_TTL_SECONDS")
|
||||||
.ok()
|
.ok()
|
||||||
.and_then(|s| s.parse::<u64>().ok())
|
.and_then(|s| s.parse::<u64>().ok())
|
||||||
.map(Duration::from_secs)
|
.map(Duration::from_secs)
|
||||||
.unwrap_or(Duration::from_secs(7 * 24 * 3600));
|
.unwrap_or(Duration::from_secs(7 * 24 * 3600));
|
||||||
|
|
||||||
let webhook_enabled = env::var("WEBHOOK")
|
let webhook_enabled = env::var("WEBHOOK")
|
||||||
.is_ok_and(|v| matches!(v.to_lowercase().as_str(), "true" | "yes" | "1"));
|
.is_ok_and(|v| matches!(v.to_lowercase().as_str(), "true" | "yes" | "1"));
|
||||||
let webhook_url = env::var("WEBHOOK_URL").ok().and_then(|s| s.parse().ok());
|
let webhook_url = env::var("WEBHOOK_URL").ok().and_then(|s| s.parse().ok());
|
||||||
let webhook_listen = env::var("WEBHOOK_LISTEN").ok().and_then(|s| s.parse().ok());
|
let webhook_listen = env::var("WEBHOOK_LISTEN").ok().and_then(|s| s.parse().ok());
|
||||||
let webhook_port = env::var("WEBHOOK_PORT").ok().and_then(|s| s.parse().ok());
|
let webhook_port = env::var("WEBHOOK_PORT").ok().and_then(|s| s.parse().ok());
|
||||||
// Empty strings count as unset (e.g. `-e WEBHOOK_CERT=` to disable a
|
// Empty strings count as unset (e.g. `-e WEBHOOK_CERT=` to disable a
|
||||||
// value that would otherwise come from `.env`).
|
// value that would otherwise come from `.env`).
|
||||||
let webhook_cert = env::var("WEBHOOK_CERT")
|
let webhook_cert = env::var("WEBHOOK_CERT").ok().filter(|s| !s.is_empty());
|
||||||
.ok()
|
let webhook_secret_token = env::var("WEBHOOK_SECRET_TOKEN")
|
||||||
.filter(|s| !s.is_empty());
|
.ok()
|
||||||
let webhook_secret_token = env::var("WEBHOOK_SECRET_TOKEN")
|
.filter(|s| !s.is_empty());
|
||||||
.ok()
|
|
||||||
.filter(|s| !s.is_empty());
|
|
||||||
|
|
||||||
Config {
|
Config {
|
||||||
admin_ids,
|
admin_ids,
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
//! Shared SQLite plumbing for the three tables in `data/task_queue.db`
|
||||||
|
//! (`tasks` in queue.rs, `chat_state` in state.rs, `link_cache` in
|
||||||
|
//! link_cache.rs).
|
||||||
|
//!
|
||||||
|
//! Every operation opens its own short-lived connection with a busy timeout:
|
||||||
|
//! handler tasks enqueue while workers lease/update rows concurrently, and
|
||||||
|
//! without the timeout a concurrent write fails immediately with SQLITE_BUSY
|
||||||
|
//! and the operation is lost. All I/O runs inside `spawn_blocking` via
|
||||||
|
//! [`with_conn`] — rusqlite connections are not Send-friendly to hold across
|
||||||
|
//! an await point, and blocking the async executor stalls every handler.
|
||||||
|
|
||||||
|
use rusqlite::Connection;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
/// Opens the shared DB with a busy timeout.
|
||||||
|
pub fn open_db(path: &str) -> rusqlite::Result<Connection> {
|
||||||
|
let conn = Connection::open(path)?;
|
||||||
|
conn.busy_timeout(Duration::from_secs(5))?;
|
||||||
|
Ok(conn)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Runs `f` against a fresh connection on a blocking thread, returning the
|
||||||
|
/// closure's result. Owns the `spawn_blocking` + `expect` ceremony shared by
|
||||||
|
/// every table access; the caller maps errors to its own log line.
|
||||||
|
pub async fn with_conn<T, F>(path: &str, f: F) -> rusqlite::Result<T>
|
||||||
|
where
|
||||||
|
T: Send + 'static,
|
||||||
|
F: FnOnce(&mut Connection) -> rusqlite::Result<T> + Send + 'static,
|
||||||
|
{
|
||||||
|
let path = path.to_string();
|
||||||
|
tokio::task::spawn_blocking(move || {
|
||||||
|
let mut conn = open_db(&path)?;
|
||||||
|
f(&mut conn)
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("db worker panicked")
|
||||||
|
}
|
||||||
@@ -5,20 +5,19 @@ use crate::send::{self, MediaItemPayload, Task};
|
|||||||
use crate::state::{ChatData, ChatStore, unix_now};
|
use crate::state::{ChatData, ChatStore, unix_now};
|
||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
use std::sync::LazyLock;
|
use std::sync::LazyLock;
|
||||||
|
use teloxide::RequestError;
|
||||||
use teloxide::prelude::*;
|
use teloxide::prelude::*;
|
||||||
use tokio::sync::Semaphore;
|
|
||||||
use teloxide::types::{
|
use teloxide::types::{
|
||||||
CallbackQuery, ChatAction, ChatId, ChatKind, InlineQuery, InlineQueryResult,
|
CallbackQuery, ChatAction, ChatId, ChatKind, InlineQuery, InlineQueryResult,
|
||||||
InlineQueryResultMpeg4Gif, InlineQueryResultPhoto, InlineQueryResultVideo, Message,
|
InlineQueryResultMpeg4Gif, InlineQueryResultPhoto, InlineQueryResultVideo, Message,
|
||||||
MessageEntityKind, MessageId, ParseMode, Recipient, ReplyParameters,
|
MessageEntityKind, MessageId, ParseMode, Recipient, ReplyParameters,
|
||||||
};
|
};
|
||||||
use teloxide::utils::command::BotCommands;
|
use teloxide::utils::command::BotCommands;
|
||||||
use teloxide::RequestError;
|
use tokio::sync::Semaphore;
|
||||||
use x_media::media::Media;
|
use x_media::media::Media;
|
||||||
|
|
||||||
pub static CHAT_STORE: LazyLock<ChatStore> = LazyLock::new(|| {
|
pub static CHAT_STORE: LazyLock<ChatStore> =
|
||||||
ChatStore::open("data/task_queue.db").expect("failed to open chat store")
|
LazyLock::new(|| ChatStore::open("data/task_queue.db").expect("failed to open chat store"));
|
||||||
});
|
|
||||||
pub static TASK_QUEUE: LazyLock<PersistentTaskQueue> =
|
pub static TASK_QUEUE: LazyLock<PersistentTaskQueue> =
|
||||||
LazyLock::new(|| PersistentTaskQueue::new("data/task_queue.db"));
|
LazyLock::new(|| PersistentTaskQueue::new("data/task_queue.db"));
|
||||||
pub static LINK_CACHE: LazyLock<LinkCache> =
|
pub static LINK_CACHE: LazyLock<LinkCache> =
|
||||||
@@ -34,24 +33,38 @@ pub static CONFIG: LazyLock<Config> = LazyLock::new(Config::load);
|
|||||||
static URL_TASKS: LazyLock<Semaphore> = LazyLock::new(|| Semaphore::new(8));
|
static URL_TASKS: LazyLock<Semaphore> = LazyLock::new(|| Semaphore::new(8));
|
||||||
|
|
||||||
#[derive(BotCommands, Clone)]
|
#[derive(BotCommands, Clone)]
|
||||||
#[command(rename_rule = "snake_case", description = "")]
|
#[command(
|
||||||
|
rename_rule = "snake_case",
|
||||||
|
description = "Turn X/Pixiv/Bluesky links into media messages"
|
||||||
|
)]
|
||||||
enum Command {
|
enum Command {
|
||||||
#[command(description = "")]
|
#[command(description = "Get started")]
|
||||||
Start,
|
Start,
|
||||||
#[command(description = "")]
|
#[command(description = "Show command help")]
|
||||||
Help,
|
Help,
|
||||||
#[command(description = "", parse_with = "split")]
|
#[command(
|
||||||
|
description = "Set forward channel (@channel or ID)",
|
||||||
|
parse_with = "split"
|
||||||
|
)]
|
||||||
SetForwardChannel(String),
|
SetForwardChannel(String),
|
||||||
#[command(description = "")]
|
#[command(description = "Remove forward channel")]
|
||||||
RemoveForwardChannel,
|
RemoveForwardChannel,
|
||||||
#[command(description = "")]
|
#[command(description = "Toggle edit-before-forward")]
|
||||||
EditBeforeForward,
|
EditBeforeForward,
|
||||||
#[command(description = "", parse_with = "split")]
|
#[command(
|
||||||
|
description = "Reply with [] to save as template",
|
||||||
|
parse_with = "split"
|
||||||
|
)]
|
||||||
SetTemplate(String),
|
SetTemplate(String),
|
||||||
#[command(description = "")]
|
#[command(description = "Show chat state (debug)")]
|
||||||
BotDict,
|
BotDict,
|
||||||
#[command(description = "", parse_with = "split")]
|
#[command(description = "Set site caption format", parse_with = "split")]
|
||||||
SetFormat(String),
|
SetFormat(String),
|
||||||
|
#[command(
|
||||||
|
description = "Clear link cache (admin; optional URL, else all)",
|
||||||
|
parse_with = "split"
|
||||||
|
)]
|
||||||
|
ClearCache(String),
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn reply<T>(bot: Bot, message: Message, text: T) -> Result<Message, RequestError>
|
async fn reply<T>(bot: Bot, message: Message, text: T) -> Result<Message, RequestError>
|
||||||
@@ -197,7 +210,11 @@ async fn set_forward_channel_handler(
|
|||||||
Ok(channel_id)
|
Ok(channel_id)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn execute_command(bot: &Bot, message: &Message, command: Command) -> Result<(), RequestError> {
|
async fn execute_command(
|
||||||
|
bot: &Bot,
|
||||||
|
message: &Message,
|
||||||
|
command: Command,
|
||||||
|
) -> Result<(), RequestError> {
|
||||||
match command {
|
match command {
|
||||||
Command::Start => {
|
Command::Start => {
|
||||||
bot.send_message(message.chat.id, "Hello!").await?;
|
bot.send_message(message.chat.id, "Hello!").await?;
|
||||||
@@ -215,7 +232,8 @@ async fn execute_command(bot: &Bot, message: &Message, command: Command) -> Resu
|
|||||||
"Add successfully.".to_string()
|
"Add successfully.".to_string()
|
||||||
}
|
}
|
||||||
Err(SetForwardChannelError::EmptyParameter) => {
|
Err(SetForwardChannelError::EmptyParameter) => {
|
||||||
"Receive empty parameter.\nYou should enter a channel id or username".to_string()
|
"Receive empty parameter.\nYou should enter a channel id or username"
|
||||||
|
.to_string()
|
||||||
}
|
}
|
||||||
Err(SetForwardChannelError::NotChannel) => {
|
Err(SetForwardChannelError::NotChannel) => {
|
||||||
"Given id / username is not a channel".to_string()
|
"Given id / username is not a channel".to_string()
|
||||||
@@ -292,7 +310,9 @@ async fn execute_command(bot: &Bot, message: &Message, command: Command) -> Resu
|
|||||||
Command::SetFormat(arg) => {
|
Command::SetFormat(arg) => {
|
||||||
let chat_id = message.chat.id.0;
|
let chat_id = message.chat.id.0;
|
||||||
let (site, format) = match arg.split_once(char::is_whitespace) {
|
let (site, format) = match arg.split_once(char::is_whitespace) {
|
||||||
Some((site, format)) if !format.trim().is_empty() => (site.trim(), format.trim().to_string()),
|
Some((site, format)) if !format.trim().is_empty() => {
|
||||||
|
(site.trim(), format.trim().to_string())
|
||||||
|
}
|
||||||
_ => {
|
_ => {
|
||||||
reply(
|
reply(
|
||||||
bot.clone(),
|
bot.clone(),
|
||||||
@@ -317,10 +337,62 @@ async fn execute_command(bot: &Bot, message: &Message, command: Command) -> Resu
|
|||||||
CHAT_STORE.set(chat_id, &chat_data).await;
|
CHAT_STORE.set(chat_id, &chat_data).await;
|
||||||
reply(bot.clone(), message.clone(), "Format set.").await?;
|
reply(bot.clone(), message.clone(), "Format set.").await?;
|
||||||
}
|
}
|
||||||
|
Command::ClearCache(arg) => {
|
||||||
|
let sender_id = message
|
||||||
|
.from
|
||||||
|
.as_ref()
|
||||||
|
.map(|user| user.id.0 as i64)
|
||||||
|
.unwrap_or(-1);
|
||||||
|
if !CONFIG.admin_ids.contains(&sender_id) {
|
||||||
|
reply(bot.clone(), message.clone(), "Admin only.").await?;
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let arg = arg.trim();
|
||||||
|
if arg.is_empty() {
|
||||||
|
let removed = LINK_CACHE.clear(None).await;
|
||||||
|
log::info!("cache cleared by {sender_id}: {removed} entries");
|
||||||
|
reply(
|
||||||
|
bot.clone(),
|
||||||
|
message.clone(),
|
||||||
|
format!("Cleared {removed} cached entr{}.", plural(removed)),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
} else {
|
||||||
|
let key = match x_media::site::cache_key(arg) {
|
||||||
|
Some(key) => key,
|
||||||
|
None => {
|
||||||
|
reply(
|
||||||
|
bot.clone(),
|
||||||
|
message.clone(),
|
||||||
|
"Unrecognized link. Use a twitter/x, pixiv or bsky post URL.",
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let removed = LINK_CACHE.clear(Some(&key)).await;
|
||||||
|
log::info!("cache entry cleared by {sender_id}: {key} ({removed} rows)");
|
||||||
|
reply(
|
||||||
|
bot.clone(),
|
||||||
|
message.clone(),
|
||||||
|
format!(
|
||||||
|
"Cleared cache for {arg} ({} entr{}).",
|
||||||
|
removed,
|
||||||
|
plural(removed)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// `""` for one, `"ies"` for anything else — "1 entry" / "2 entries".
|
||||||
|
fn plural(n: usize) -> &'static str {
|
||||||
|
if n == 1 { "" } else { "ies" }
|
||||||
|
}
|
||||||
|
|
||||||
/// For locally produced media (encoded ugoira MP4) the thumbnail URL is a
|
/// For locally produced media (encoded ugoira MP4) the thumbnail URL is a
|
||||||
/// hotlink-protected remote URL Telegram may not fetch; let Telegram generate
|
/// hotlink-protected remote URL Telegram may not fetch; let Telegram generate
|
||||||
/// its own thumbnail instead.
|
/// its own thumbnail instead.
|
||||||
@@ -383,7 +455,10 @@ async fn dispatch_send(bot: Bot, message: &Message, task: &Task, url: &str) {
|
|||||||
log::info!("sent {} message(s) for {url}", message_ids.len());
|
log::info!("sent {} message(s) for {url}", message_ids.len());
|
||||||
send::post_send_actions(&bot, task, message_ids).await;
|
send::post_send_actions(&bot, task, message_ids).await;
|
||||||
}
|
}
|
||||||
Err(send::SendError::Retryable { delay_seconds, task }) => {
|
Err(send::SendError::Retryable {
|
||||||
|
delay_seconds,
|
||||||
|
task,
|
||||||
|
}) => {
|
||||||
log::info!("send for {url} failed, queued for retry in {delay_seconds:.1}s");
|
log::info!("send for {url} failed, queued for retry in {delay_seconds:.1}s");
|
||||||
enqueue_retry(task, delay_seconds).await;
|
enqueue_retry(task, delay_seconds).await;
|
||||||
let _ = reply(bot, message.clone(), "Send failed. Task queued for retry.").await;
|
let _ = reply(bot, message.clone(), "Send failed. Task queued for retry.").await;
|
||||||
@@ -444,7 +519,10 @@ fn build_send_task(
|
|||||||
|
|
||||||
async fn url_media(bot: Bot, message: &Message, url: &str) {
|
async fn url_media(bot: Bot, message: &Message, url: &str) {
|
||||||
let chat_id = message.chat.id.0;
|
let chat_id = message.chat.id.0;
|
||||||
if let Err(e) = bot.send_chat_action(ChatId(chat_id), ChatAction::Typing).await {
|
if let Err(e) = bot
|
||||||
|
.send_chat_action(ChatId(chat_id), ChatAction::Typing)
|
||||||
|
.await
|
||||||
|
{
|
||||||
log::error!("send_chat_action failed: {e}");
|
log::error!("send_chat_action failed: {e}");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -520,7 +598,12 @@ async fn url_media(bot: Bot, message: &Message, url: &str) {
|
|||||||
// Retries exhausted: notify the user (Rust-only requirement 3).
|
// Retries exhausted: notify the user (Rust-only requirement 3).
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
log::error!("fetch {url}: {e}");
|
log::error!("fetch {url}: {e}");
|
||||||
let _ = reply(bot, message.clone(), "Failed to fetch media from this link.").await;
|
let _ = reply(
|
||||||
|
bot,
|
||||||
|
message.clone(),
|
||||||
|
"Failed to fetch media from this link.",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
Ok(Some(fetched)) => {
|
Ok(Some(fetched)) => {
|
||||||
if fetched.media.is_empty() {
|
if fetched.media.is_empty() {
|
||||||
@@ -542,8 +625,9 @@ async fn url_media(bot: Bot, message: &Message, url: &str) {
|
|||||||
let caption = fetched.caption_with(&format);
|
let caption = fetched.caption_with(&format);
|
||||||
// Raw render data for the link cache; the send fills in the
|
// Raw render data for the link cache; the send fills in the
|
||||||
// Telegram file ids and persists the entry.
|
// Telegram file ids and persists the entry.
|
||||||
let cache_data = fetched.render_fields().map(|(author, author_url, title, tags)| {
|
let cache_data = fetched
|
||||||
CachedPost {
|
.render_fields()
|
||||||
|
.map(|(author, author_url, title, tags)| CachedPost {
|
||||||
url: fetched.source_url.clone(),
|
url: fetched.source_url.clone(),
|
||||||
caption: fetched.caption.clone(),
|
caption: fetched.caption.clone(),
|
||||||
title: title.to_string(),
|
title: title.to_string(),
|
||||||
@@ -552,8 +636,7 @@ async fn url_media(bot: Bot, message: &Message, url: &str) {
|
|||||||
tags: tags.to_string(),
|
tags: tags.to_string(),
|
||||||
sensitive: fetched.sensitive,
|
sensitive: fetched.sensitive,
|
||||||
media: vec![],
|
media: vec![],
|
||||||
}
|
});
|
||||||
});
|
|
||||||
let items: Vec<MediaItemPayload> = fetched
|
let items: Vec<MediaItemPayload> = fetched
|
||||||
.media
|
.media
|
||||||
.iter()
|
.iter()
|
||||||
@@ -583,7 +666,10 @@ pub async fn message_handler(bot: Bot, message: Message) -> Result<(), RequestEr
|
|||||||
.text()
|
.text()
|
||||||
.map(|t| if t.len() > 120 { &t[..120] } else { t })
|
.map(|t| if t.len() > 120 { &t[..120] } else { t })
|
||||||
.unwrap_or("<no text>");
|
.unwrap_or("<no text>");
|
||||||
log::info!("message from {sender} in {} (private={is_private}): {text_preview}", message.chat.id);
|
log::info!(
|
||||||
|
"message from {sender} in {} (private={is_private}): {text_preview}",
|
||||||
|
message.chat.id
|
||||||
|
);
|
||||||
// URL/edit flows only run in private chats; commands run in any chat.
|
// URL/edit flows only run in private chats; commands run in any chat.
|
||||||
if is_private && edit_message_handler(&bot, &message).await {
|
if is_private && edit_message_handler(&bot, &message).await {
|
||||||
return respond(());
|
return respond(());
|
||||||
@@ -653,8 +739,8 @@ pub async fn inline_query_handler(bot: Bot, query: InlineQuery) -> Result<(), Re
|
|||||||
thumbnail,
|
thumbnail,
|
||||||
fetched.title.clone(),
|
fetched.title.clone(),
|
||||||
)
|
)
|
||||||
.caption(caption)
|
.caption(caption)
|
||||||
.parse_mode(ParseMode::Html),
|
.parse_mode(ParseMode::Html),
|
||||||
),
|
),
|
||||||
Media::Animated { .. } => InlineQueryResult::Mpeg4Gif(
|
Media::Animated { .. } => InlineQueryResult::Mpeg4Gif(
|
||||||
InlineQueryResultMpeg4Gif::new(id, url, thumbnail)
|
InlineQueryResultMpeg4Gif::new(id, url, thumbnail)
|
||||||
@@ -686,7 +772,10 @@ pub async fn callback_query_handler(bot: Bot, query: CallbackQuery) -> Result<()
|
|||||||
let mut chat_data = CHAT_STORE.get(chat_id).await;
|
let mut chat_data = CHAT_STORE.get(chat_id).await;
|
||||||
let edit = chat_data.edit_message.get(&prompt_message_id).cloned();
|
let edit = chat_data.edit_message.get(&prompt_message_id).cloned();
|
||||||
let Some(edit) = edit else {
|
let Some(edit) = edit else {
|
||||||
log::info!("callback from {}: no edit record for prompt {prompt_message_id}", chat_id);
|
log::info!(
|
||||||
|
"callback from {}: no edit record for prompt {prompt_message_id}",
|
||||||
|
chat_id
|
||||||
|
);
|
||||||
bot.answer_callback_query(callback_query_id)
|
bot.answer_callback_query(callback_query_id)
|
||||||
.text("Expired")
|
.text("Expired")
|
||||||
.await?;
|
.await?;
|
||||||
@@ -705,7 +794,10 @@ pub async fn callback_query_handler(bot: Bot, query: CallbackQuery) -> Result<()
|
|||||||
let Some(data) = data else {
|
let Some(data) = data else {
|
||||||
return respond(());
|
return respond(());
|
||||||
};
|
};
|
||||||
log::info!("callback from {} on prompt {prompt_message_id}: {data}", chat_id);
|
log::info!(
|
||||||
|
"callback from {} on prompt {prompt_message_id}: {data}",
|
||||||
|
chat_id
|
||||||
|
);
|
||||||
if data == "forward" {
|
if data == "forward" {
|
||||||
match chat_data.forward_channel_id {
|
match chat_data.forward_channel_id {
|
||||||
Some(channel_id) => {
|
Some(channel_id) => {
|
||||||
@@ -731,7 +823,10 @@ pub async fn callback_query_handler(bot: Bot, query: CallbackQuery) -> Result<()
|
|||||||
chat_data.edit_message.remove(&prompt_message_id);
|
chat_data.edit_message.remove(&prompt_message_id);
|
||||||
CHAT_STORE.set(chat_id, &chat_data).await;
|
CHAT_STORE.set(chat_id, &chat_data).await;
|
||||||
}
|
}
|
||||||
Err(send::SendError::Retryable { delay_seconds, task }) => {
|
Err(send::SendError::Retryable {
|
||||||
|
delay_seconds,
|
||||||
|
task,
|
||||||
|
}) => {
|
||||||
log::info!("forward queued for retry in {delay_seconds:.1}s");
|
log::info!("forward queued for retry in {delay_seconds:.1}s");
|
||||||
enqueue_retry(task, delay_seconds).await;
|
enqueue_retry(task, delay_seconds).await;
|
||||||
bot.answer_callback_query(callback_query_id)
|
bot.answer_callback_query(callback_query_id)
|
||||||
|
|||||||
@@ -8,7 +8,7 @@
|
|||||||
//! [`Config::link_cache_ttl`]; a stale entry is dropped lazily on read and
|
//! [`Config::link_cache_ttl`]; a stale entry is dropped lazily on read and
|
||||||
//! by the periodic prune in `main`.
|
//! by the periodic prune in `main`.
|
||||||
|
|
||||||
use rusqlite::{params, Connection};
|
use rusqlite::{Connection, params};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
@@ -49,12 +49,6 @@ pub struct LinkCache {
|
|||||||
db_path: String,
|
db_path: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn open_db(path: &str) -> rusqlite::Result<Connection> {
|
|
||||||
let conn = Connection::open(path)?;
|
|
||||||
conn.busy_timeout(Duration::from_secs(5))?;
|
|
||||||
Ok(conn)
|
|
||||||
}
|
|
||||||
|
|
||||||
impl LinkCache {
|
impl LinkCache {
|
||||||
pub fn open(db_path: &str) -> Self {
|
pub fn open(db_path: &str) -> Self {
|
||||||
if let Ok(conn) = Connection::open(db_path)
|
if let Ok(conn) = Connection::open(db_path)
|
||||||
@@ -73,11 +67,9 @@ impl LinkCache {
|
|||||||
/// Returns the cached post if present and not expired; a stale entry is
|
/// Returns the cached post if present and not expired; a stale entry is
|
||||||
/// removed on the spot.
|
/// removed on the spot.
|
||||||
pub async fn get(&self, key: &str, ttl: Duration) -> Option<CachedPost> {
|
pub async fn get(&self, key: &str, ttl: Duration) -> Option<CachedPost> {
|
||||||
let db_path = self.db_path.clone();
|
|
||||||
let key = key.to_string();
|
let key = key.to_string();
|
||||||
let ttl = ttl.as_secs_f64();
|
let ttl = ttl.as_secs_f64();
|
||||||
tokio::task::spawn_blocking(move || -> rusqlite::Result<Option<CachedPost>> {
|
let result = crate::db::with_conn(&self.db_path, move |conn| {
|
||||||
let conn = open_db(&db_path)?;
|
|
||||||
let mut stmt =
|
let mut stmt =
|
||||||
conn.prepare("SELECT payload, created_at FROM link_cache WHERE url = ?1")?;
|
conn.prepare("SELECT payload, created_at FROM link_cache WHERE url = ?1")?;
|
||||||
let mut rows = stmt.query(params![key])?;
|
let mut rows = stmt.query(params![key])?;
|
||||||
@@ -90,66 +82,84 @@ impl LinkCache {
|
|||||||
conn.execute("DELETE FROM link_cache WHERE url = ?1", params![key])?;
|
conn.execute("DELETE FROM link_cache WHERE url = ?1", params![key])?;
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
serde_json::from_str(&payload).map(Some).map_err(|e| {
|
Ok(Some(serde_json::from_str::<CachedPost>(&payload).map_err(
|
||||||
rusqlite::Error::ToSqlConversionFailure(Box::new(e))
|
|e| rusqlite::Error::ToSqlConversionFailure(Box::new(e)),
|
||||||
})
|
)?))
|
||||||
})
|
|
||||||
.await
|
|
||||||
.expect("link cache read worker panicked")
|
|
||||||
.unwrap_or_else(|e| {
|
|
||||||
log::error!("link cache read failed: {e}");
|
|
||||||
None
|
|
||||||
})
|
})
|
||||||
|
.await;
|
||||||
|
match result {
|
||||||
|
Ok(v) => v,
|
||||||
|
Err(e) => {
|
||||||
|
log::error!("link cache read failed: {e}");
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn put(&self, key: &str, post: &CachedPost) {
|
pub async fn put(&self, key: &str, post: &CachedPost) {
|
||||||
let db_path = self.db_path.clone();
|
|
||||||
let key = key.to_string();
|
let key = key.to_string();
|
||||||
let payload = serde_json::to_string(post).expect("cached post serializes");
|
let payload = serde_json::to_string(post).expect("cached post serializes");
|
||||||
tokio::task::spawn_blocking(move || -> rusqlite::Result<()> {
|
let result = crate::db::with_conn(&self.db_path, move |conn| {
|
||||||
let conn = open_db(&db_path)?;
|
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"INSERT OR REPLACE INTO link_cache (url, payload, created_at) VALUES (?1, ?2, ?3)",
|
"INSERT OR REPLACE INTO link_cache (url, payload, created_at) VALUES (?1, ?2, ?3)",
|
||||||
params![key, payload, now_f64()],
|
params![key, payload, now_f64()],
|
||||||
)?;
|
)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
.expect("link cache write worker panicked")
|
if let Err(e) = result {
|
||||||
.unwrap_or_else(|e| log::error!("link cache write failed: {e}"));
|
log::error!("link cache write failed: {e}");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Drops an entry (e.g. a cached file id that turned out invalid).
|
/// Drops an entry (e.g. a cached file id that turned out invalid).
|
||||||
pub async fn remove(&self, key: &str) {
|
pub async fn remove(&self, key: &str) {
|
||||||
let db_path = self.db_path.clone();
|
|
||||||
let key = key.to_string();
|
let key = key.to_string();
|
||||||
tokio::task::spawn_blocking(move || -> rusqlite::Result<()> {
|
let result = crate::db::with_conn(&self.db_path, move |conn| {
|
||||||
let conn = open_db(&db_path)?;
|
|
||||||
conn.execute("DELETE FROM link_cache WHERE url = ?1", params![key])?;
|
conn.execute("DELETE FROM link_cache WHERE url = ?1", params![key])?;
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
.expect("link cache delete worker panicked")
|
if let Err(e) = result {
|
||||||
.unwrap_or_else(|e| log::error!("link cache delete failed: {e}"));
|
log::error!("link cache delete failed: {e}");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Removes expired entries; returns how many were deleted.
|
/// Removes expired entries; returns how many were deleted.
|
||||||
pub async fn prune(&self, ttl: Duration) -> usize {
|
pub async fn prune(&self, ttl: Duration) -> usize {
|
||||||
let db_path = self.db_path.clone();
|
|
||||||
let cutoff = now_f64() - ttl.as_secs_f64();
|
let cutoff = now_f64() - ttl.as_secs_f64();
|
||||||
tokio::task::spawn_blocking(move || -> rusqlite::Result<usize> {
|
let result = crate::db::with_conn(&self.db_path, move |conn| {
|
||||||
let conn = open_db(&db_path)?;
|
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"DELETE FROM link_cache WHERE created_at < ?1",
|
"DELETE FROM link_cache WHERE created_at < ?1",
|
||||||
params![cutoff],
|
params![cutoff],
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
.expect("link cache prune worker panicked")
|
match result {
|
||||||
.unwrap_or_else(|e| {
|
Ok(n) => n,
|
||||||
log::error!("link cache prune failed: {e}");
|
Err(e) => {
|
||||||
0
|
log::error!("link cache prune failed: {e}");
|
||||||
|
0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Deletes one entry (by normalized cache key) or the whole cache when
|
||||||
|
/// `key` is `None`. Returns how many rows were removed.
|
||||||
|
pub async fn clear(&self, key: Option<&str>) -> usize {
|
||||||
|
let key = key.map(str::to_string);
|
||||||
|
let result = crate::db::with_conn(&self.db_path, move |conn| match &key {
|
||||||
|
Some(key) => conn.execute("DELETE FROM link_cache WHERE url = ?1", params![key]),
|
||||||
|
None => conn.execute("DELETE FROM link_cache", []),
|
||||||
})
|
})
|
||||||
|
.await;
|
||||||
|
match result {
|
||||||
|
Ok(n) => n,
|
||||||
|
Err(e) => {
|
||||||
|
log::error!("link cache clear failed: {e}");
|
||||||
|
0
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -200,14 +210,21 @@ mod tests {
|
|||||||
// Force the row into the past so a 1s TTL expires it.
|
// Force the row into the past so a 1s TTL expires it.
|
||||||
{
|
{
|
||||||
let conn = Connection::open(dir.path().join("c.db")).unwrap();
|
let conn = Connection::open(dir.path().join("c.db")).unwrap();
|
||||||
conn.execute(
|
conn.execute("UPDATE link_cache SET created_at = created_at - 100", [])
|
||||||
"UPDATE link_cache SET created_at = created_at - 100",
|
.unwrap();
|
||||||
[],
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
}
|
}
|
||||||
assert!(cache.get("twitter:1", Duration::from_secs(1)).await.is_none());
|
assert!(
|
||||||
assert!(cache.get("twitter:1", Duration::from_secs(3600)).await.is_none());
|
cache
|
||||||
|
.get("twitter:1", Duration::from_secs(1))
|
||||||
|
.await
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
cache
|
||||||
|
.get("twitter:1", Duration::from_secs(3600))
|
||||||
|
.await
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -217,14 +234,60 @@ mod tests {
|
|||||||
cache.put("twitter:1", &entry()).await;
|
cache.put("twitter:1", &entry()).await;
|
||||||
cache.put("pixiv:2", &entry()).await;
|
cache.put("pixiv:2", &entry()).await;
|
||||||
cache.remove("twitter:1").await;
|
cache.remove("twitter:1").await;
|
||||||
assert!(cache.get("twitter:1", Duration::from_secs(3600)).await.is_none());
|
assert!(
|
||||||
assert!(cache.get("pixiv:2", Duration::from_secs(3600)).await.is_some());
|
cache
|
||||||
|
.get("twitter:1", Duration::from_secs(3600))
|
||||||
|
.await
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
cache
|
||||||
|
.get("pixiv:2", Duration::from_secs(3600))
|
||||||
|
.await
|
||||||
|
.is_some()
|
||||||
|
);
|
||||||
{
|
{
|
||||||
let conn = Connection::open(dir.path().join("c.db")).unwrap();
|
let conn = Connection::open(dir.path().join("c.db")).unwrap();
|
||||||
conn.execute("UPDATE link_cache SET created_at = created_at - 100", [])
|
conn.execute("UPDATE link_cache SET created_at = created_at - 100", [])
|
||||||
.unwrap();
|
.unwrap();
|
||||||
}
|
}
|
||||||
assert_eq!(cache.prune(Duration::from_secs(1)).await, 1);
|
assert_eq!(cache.prune(Duration::from_secs(1)).await, 1);
|
||||||
assert!(cache.get("pixiv:2", Duration::from_secs(3600)).await.is_none());
|
assert!(
|
||||||
|
cache
|
||||||
|
.get("pixiv:2", Duration::from_secs(3600))
|
||||||
|
.await
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn clear_one_entry_or_all() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let cache = LinkCache::open(dir.path().join("c.db").to_str().unwrap());
|
||||||
|
cache.put("twitter:1", &entry()).await;
|
||||||
|
cache.put("pixiv:2", &entry()).await;
|
||||||
|
// By key: only the matching row is removed.
|
||||||
|
assert_eq!(cache.clear(Some("twitter:1")).await, 1);
|
||||||
|
assert!(
|
||||||
|
cache
|
||||||
|
.get("twitter:1", Duration::from_secs(3600))
|
||||||
|
.await
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
cache
|
||||||
|
.get("pixiv:2", Duration::from_secs(3600))
|
||||||
|
.await
|
||||||
|
.is_some()
|
||||||
|
);
|
||||||
|
// Whole cache: nothing left; removing an absent key deletes 0 rows.
|
||||||
|
assert_eq!(cache.clear(None).await, 1);
|
||||||
|
assert!(
|
||||||
|
cache
|
||||||
|
.get("pixiv:2", Duration::from_secs(3600))
|
||||||
|
.await
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert_eq!(cache.clear(None).await, 0);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,13 +1,14 @@
|
|||||||
use dotenv::dotenv;
|
use dotenv::dotenv;
|
||||||
use teloxide::dptree::endpoint;
|
use teloxide::dptree::endpoint;
|
||||||
|
use teloxide::prelude::*;
|
||||||
use teloxide::stop::StopToken;
|
use teloxide::stop::StopToken;
|
||||||
use teloxide::types::{ChatId, InputFile, MessageId};
|
use teloxide::types::{ChatId, InputFile, MessageId};
|
||||||
use teloxide::update_listeners::{self, webhooks, UpdateListener};
|
use teloxide::update_listeners::{self, UpdateListener, webhooks};
|
||||||
use teloxide::prelude::*;
|
|
||||||
use tokio::sync::watch;
|
use tokio::sync::watch;
|
||||||
use x_media::site;
|
use x_media::site;
|
||||||
|
|
||||||
mod config;
|
mod config;
|
||||||
|
mod db;
|
||||||
mod handlers;
|
mod handlers;
|
||||||
mod link_cache;
|
mod link_cache;
|
||||||
mod photo;
|
mod photo;
|
||||||
@@ -74,7 +75,10 @@ async fn main() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Edit-expiry sweep: clears the prompt's buttons once the record expires.
|
// Edit-expiry sweep: clears the prompt's buttons once the record expires.
|
||||||
log::info!("edit-expiry sweep: every 300s, ttl {}", CONFIG.edit_message_ttl.as_secs());
|
log::info!(
|
||||||
|
"edit-expiry sweep: every 300s, ttl {}",
|
||||||
|
CONFIG.edit_message_ttl.as_secs()
|
||||||
|
);
|
||||||
let (stop_tx, stop_rx) = watch::channel(false);
|
let (stop_tx, stop_rx) = watch::channel(false);
|
||||||
{
|
{
|
||||||
let bot = bot.clone();
|
let bot = bot.clone();
|
||||||
@@ -95,7 +99,10 @@ async fn main() {
|
|||||||
// If the prompt was already deleted, this fails with a
|
// If the prompt was already deleted, this fails with a
|
||||||
// 400 "message to edit not found" — log and ignore.
|
// 400 "message to edit not found" — log and ignore.
|
||||||
if let Err(e) = bot
|
if let Err(e) = bot
|
||||||
.edit_message_reply_markup(ChatId(chat_id), MessageId(prompt_message_id as i32))
|
.edit_message_reply_markup(
|
||||||
|
ChatId(chat_id),
|
||||||
|
MessageId(prompt_message_id as i32),
|
||||||
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
log::info!("edit-expiry sweep: prompt message gone: {e}");
|
log::info!("edit-expiry sweep: prompt message gone: {e}");
|
||||||
@@ -117,10 +124,7 @@ async fn main() {
|
|||||||
|
|
||||||
if CONFIG.webhook_enabled {
|
if CONFIG.webhook_enabled {
|
||||||
log::info!("running in webhook mode");
|
log::info!("running in webhook mode");
|
||||||
let url = CONFIG
|
let url = CONFIG.webhook_url.clone().expect("WEBHOOK_URL is not set");
|
||||||
.webhook_url
|
|
||||||
.clone()
|
|
||||||
.expect("WEBHOOK_URL is not set");
|
|
||||||
// `webhooks::axum` calls set_webhook itself (with the full options,
|
// `webhooks::axum` calls set_webhook itself (with the full options,
|
||||||
// secret token included) — no explicit registration here.
|
// secret token included) — no explicit registration here.
|
||||||
let listen = CONFIG.webhook_listen.expect("WEBHOOK_LISTEN is not set");
|
let listen = CONFIG.webhook_listen.expect("WEBHOOK_LISTEN is not set");
|
||||||
|
|||||||
@@ -216,13 +216,15 @@ fn target_dims(w: u32, h: u32) -> (u32, u32) {
|
|||||||
/// RGB, Lanczos-downscale beyond the dimension cap, encode PNG — a PNG still
|
/// RGB, Lanczos-downscale beyond the dimension cap, encode PNG — a PNG still
|
||||||
/// over the upload cap afterwards becomes JPEG.
|
/// over the upload cap afterwards becomes JPEG.
|
||||||
fn prepare_png(file: NamedTempFile, bytes: Vec<u8>) -> Result<PhotoPrep, String> {
|
fn prepare_png(file: NamedTempFile, bytes: Vec<u8>) -> Result<PhotoPrep, String> {
|
||||||
let (w, h, _bit_depth, color_type) =
|
let (w, h, _bit_depth, color_type) = parse_png_header(&bytes).ok_or("invalid PNG header")?;
|
||||||
parse_png_header(&bytes).ok_or("invalid PNG header")?;
|
|
||||||
let size_over = bytes.len() as u64 > MAX_UPLOAD_BYTES;
|
let size_over = bytes.len() as u64 > MAX_UPLOAD_BYTES;
|
||||||
if w + h <= PHOTO_MAX_DIMENSION_SUM && !size_over {
|
if w + h <= PHOTO_MAX_DIMENSION_SUM && !size_over {
|
||||||
return Ok(PhotoPrep::Upload(file));
|
return Ok(PhotoPrep::Upload(file));
|
||||||
}
|
}
|
||||||
log::info!("photo {w}x{h} ({_bit_depth:?} {color_type:?}, {} bytes) needs processing", bytes.len());
|
log::info!(
|
||||||
|
"photo {w}x{h} ({_bit_depth:?} {color_type:?}, {} bytes) needs processing",
|
||||||
|
bytes.len()
|
||||||
|
);
|
||||||
|
|
||||||
let channels = output_channels(color_type);
|
let channels = output_channels(color_type);
|
||||||
if (w as u64) * (h as u64) * channels as u64 > MAX_DECODE_BYTES {
|
if (w as u64) * (h as u64) * channels as u64 > MAX_DECODE_BYTES {
|
||||||
@@ -238,7 +240,9 @@ fn prepare_png(file: NamedTempFile, bytes: Vec<u8>) -> Result<PhotoPrep, String>
|
|||||||
};
|
};
|
||||||
let mut decoder = png::Decoder::new(std::io::Cursor::new(&bytes));
|
let mut decoder = png::Decoder::new(std::io::Cursor::new(&bytes));
|
||||||
decoder.set_transformations(transforms);
|
decoder.set_transformations(transforms);
|
||||||
let mut reader = decoder.read_info().map_err(|e| format!("png decode: {e}"))?;
|
let mut reader = decoder
|
||||||
|
.read_info()
|
||||||
|
.map_err(|e| format!("png decode: {e}"))?;
|
||||||
let out_w = reader.info().width;
|
let out_w = reader.info().width;
|
||||||
let out_h = reader.info().height;
|
let out_h = reader.info().height;
|
||||||
let mut buf = vec![
|
let mut buf = vec![
|
||||||
@@ -457,7 +461,9 @@ mod tests {
|
|||||||
let mut bytes = Vec::new();
|
let mut bytes = Vec::new();
|
||||||
{
|
{
|
||||||
let encoder = jpeg_encoder::Encoder::new(&mut bytes, 90);
|
let encoder = jpeg_encoder::Encoder::new(&mut bytes, 90);
|
||||||
encoder.encode(&rgb, w, h, jpeg_encoder::ColorType::Rgb).unwrap();
|
encoder
|
||||||
|
.encode(&rgb, w, h, jpeg_encoder::ColorType::Rgb)
|
||||||
|
.unwrap();
|
||||||
}
|
}
|
||||||
let mut file = tempfile::Builder::new().suffix(".jpg").tempfile().unwrap();
|
let mut file = tempfile::Builder::new().suffix(".jpg").tempfile().unwrap();
|
||||||
std::io::Write::write_all(file.as_file_mut(), &bytes).unwrap();
|
std::io::Write::write_all(file.as_file_mut(), &bytes).unwrap();
|
||||||
@@ -485,7 +491,9 @@ mod tests {
|
|||||||
for y in 0..h {
|
for y in 0..h {
|
||||||
for x in 0..w {
|
for x in 0..w {
|
||||||
let base = (x + y) * 255 / (w + h);
|
let base = (x + y) * 255 / (w + h);
|
||||||
rng = rng.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
|
rng = rng
|
||||||
|
.wrapping_mul(6364136223846793005)
|
||||||
|
.wrapping_add(1442695040888963407);
|
||||||
let n = ((rng >> 33) % 11) as i32 - 5; // noise in [-5, 5]
|
let n = ((rng >> 33) % 11) as i32 - 5; // noise in [-5, 5]
|
||||||
let v = (base as i32 + n).clamp(0, 255) as u8;
|
let v = (base as i32 + n).clamp(0, 255) as u8;
|
||||||
data.extend_from_slice(&[v, v, v]);
|
data.extend_from_slice(&[v, v, v]);
|
||||||
@@ -499,7 +507,11 @@ mod tests {
|
|||||||
let mut writer = encoder.write_header().unwrap();
|
let mut writer = encoder.write_header().unwrap();
|
||||||
writer.write_image_data(&data).unwrap();
|
writer.write_image_data(&data).unwrap();
|
||||||
}
|
}
|
||||||
assert!(bytes.len() as u64 > MAX_UPLOAD_BYTES, "test needs a >10MiB PNG, got {}", bytes.len());
|
assert!(
|
||||||
|
bytes.len() as u64 > MAX_UPLOAD_BYTES,
|
||||||
|
"test needs a >10MiB PNG, got {}",
|
||||||
|
bytes.len()
|
||||||
|
);
|
||||||
|
|
||||||
let mut file = tempfile::Builder::new().suffix(".png").tempfile().unwrap();
|
let mut file = tempfile::Builder::new().suffix(".png").tempfile().unwrap();
|
||||||
std::io::Write::write_all(file.as_file_mut(), &bytes).unwrap();
|
std::io::Write::write_all(file.as_file_mut(), &bytes).unwrap();
|
||||||
|
|||||||
@@ -6,11 +6,11 @@
|
|||||||
//! replaced by dedicated columns.
|
//! replaced by dedicated columns.
|
||||||
|
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
use rusqlite::{params, Connection, TransactionBehavior};
|
use rusqlite::{Connection, TransactionBehavior, params};
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use std::pin::Pin;
|
use std::pin::Pin;
|
||||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||||
use tokio::sync::Notify;
|
use tokio::sync::Notify;
|
||||||
use tokio::task::JoinHandle;
|
use tokio::task::JoinHandle;
|
||||||
@@ -29,15 +29,9 @@ const QUEUE_WORKERS: usize = 4;
|
|||||||
pub enum QueueError {
|
pub enum QueueError {
|
||||||
/// Reschedule with the given delay; after `MAX_RETRIES` attempts the task
|
/// Reschedule with the given delay; after `MAX_RETRIES` attempts the task
|
||||||
/// is dead-lettered instead.
|
/// is dead-lettered instead.
|
||||||
Retryable {
|
Retryable { delay_seconds: f64, payload: Value },
|
||||||
delay_seconds: f64,
|
|
||||||
payload: Value,
|
|
||||||
},
|
|
||||||
/// Give up now.
|
/// Give up now.
|
||||||
Permanent {
|
Permanent { message: String, payload: Value },
|
||||||
message: String,
|
|
||||||
payload: Value,
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type BoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
|
type BoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
|
||||||
@@ -74,16 +68,7 @@ fn now_f64() -> f64 {
|
|||||||
.unwrap_or(0.0)
|
.unwrap_or(0.0)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Opens the queue DB with a busy timeout. Handler tasks enqueue while
|
fn ensure_schema(conn: &rusqlite::Connection) -> rusqlite::Result<()> {
|
||||||
/// workers lease/update rows concurrently; without the timeout a concurrent
|
|
||||||
/// write fails immediately with SQLITE_BUSY and the operation is lost.
|
|
||||||
fn open_db(path: &str) -> rusqlite::Result<Connection> {
|
|
||||||
let conn = Connection::open(path)?;
|
|
||||||
conn.busy_timeout(Duration::from_secs(5))?;
|
|
||||||
Ok(conn)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn ensure_schema(conn: &Connection) -> rusqlite::Result<()> {
|
|
||||||
conn.execute_batch(
|
conn.execute_batch(
|
||||||
"CREATE TABLE IF NOT EXISTS tasks (id TEXT PRIMARY KEY, payload TEXT NOT NULL, \
|
"CREATE TABLE IF NOT EXISTS tasks (id TEXT PRIMARY KEY, payload TEXT NOT NULL, \
|
||||||
run_after REAL NOT NULL, attempts INTEGER NOT NULL, status TEXT NOT NULL, \
|
run_after REAL NOT NULL, attempts INTEGER NOT NULL, status TEXT NOT NULL, \
|
||||||
@@ -162,10 +147,8 @@ impl PersistentTaskQueue {
|
|||||||
self.counter.fetch_add(1, Ordering::Relaxed)
|
self.counter.fetch_add(1, Ordering::Relaxed)
|
||||||
);
|
);
|
||||||
let payload = payload.to_string();
|
let payload = payload.to_string();
|
||||||
let db_path = self.db_path.clone();
|
|
||||||
log::info!("enqueued {id} (run_after {run_after:.1})");
|
log::info!("enqueued {id} (run_after {run_after:.1})");
|
||||||
tokio::task::spawn_blocking(move || -> rusqlite::Result<()> {
|
crate::db::with_conn(&self.db_path, move |conn| {
|
||||||
let conn = open_db(&db_path)?;
|
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"INSERT OR REPLACE INTO tasks (id, payload, run_after, attempts, status, locked_until, created_at) \
|
"INSERT OR REPLACE INTO tasks (id, payload, run_after, attempts, status, locked_until, created_at) \
|
||||||
VALUES (?1, ?2, ?3, 0, 'pending', 0, ?4)",
|
VALUES (?1, ?2, ?3, 0, 'pending', 0, ?4)",
|
||||||
@@ -173,8 +156,7 @@ impl PersistentTaskQueue {
|
|||||||
)?;
|
)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
.await
|
.await?;
|
||||||
.expect("queue insert worker panicked")?;
|
|
||||||
// Wake every sleeping worker: with several workers the one that finds
|
// Wake every sleeping worker: with several workers the one that finds
|
||||||
// nothing due must not starve the newly inserted row.
|
// nothing due must not starve the newly inserted row.
|
||||||
self.notify.notify_waiters();
|
self.notify.notify_waiters();
|
||||||
@@ -182,18 +164,17 @@ impl PersistentTaskQueue {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn recover_stale(&self) {
|
async fn recover_stale(&self) {
|
||||||
let db_path = self.db_path.clone();
|
let result = crate::db::with_conn(&self.db_path, move |conn| {
|
||||||
tokio::task::spawn_blocking(move || -> rusqlite::Result<()> {
|
|
||||||
let conn = open_db(&db_path)?;
|
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"UPDATE tasks SET status='pending', locked_until=0 WHERE status='in_progress' AND locked_until < ?1",
|
"UPDATE tasks SET status='pending', locked_until=0 WHERE status='in_progress' AND locked_until < ?1",
|
||||||
params![now_f64()],
|
params![now_f64()],
|
||||||
)?;
|
)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
.expect("queue recovery worker panicked")
|
if let Err(e) = result {
|
||||||
.unwrap_or_else(|e| log::error!("queue recovery failed: {e}"));
|
log::error!("queue recovery failed: {e}");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -225,9 +206,7 @@ impl QueueWorker {
|
|||||||
|
|
||||||
/// Leases the oldest due row (sets it `in_progress` with a lock TTL).
|
/// Leases the oldest due row (sets it `in_progress` with a lock TTL).
|
||||||
async fn lease_next(&self) -> Option<LeasedRow> {
|
async fn lease_next(&self) -> Option<LeasedRow> {
|
||||||
let db_path = self.db_path.clone();
|
let result = crate::db::with_conn(&self.db_path, |conn| {
|
||||||
tokio::task::spawn_blocking(move || -> rusqlite::Result<Option<LeasedRow>> {
|
|
||||||
let mut conn = open_db(&db_path)?;
|
|
||||||
// BEGIN IMMEDIATE: with several workers, a deferred transaction
|
// BEGIN IMMEDIATE: with several workers, a deferred transaction
|
||||||
// that read before another worker's lease commit would fail with
|
// that read before another worker's lease commit would fail with
|
||||||
// SQLITE_BUSY_SNAPSHOT. Taking the write lock up front serializes
|
// SQLITE_BUSY_SNAPSHOT. Taking the write lock up front serializes
|
||||||
@@ -265,31 +244,34 @@ impl QueueWorker {
|
|||||||
attempts,
|
attempts,
|
||||||
}))
|
}))
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
.expect("queue lease worker panicked")
|
match result {
|
||||||
.unwrap_or_else(|e| {
|
Ok(row) => row,
|
||||||
log::error!("queue lease failed: {e}");
|
Err(e) => {
|
||||||
None
|
log::error!("queue lease failed: {e}");
|
||||||
})
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn earliest_run_after(&self) -> Option<f64> {
|
async fn earliest_run_after(&self) -> Option<f64> {
|
||||||
let db_path = self.db_path.clone();
|
let result = crate::db::with_conn(&self.db_path, |conn| {
|
||||||
tokio::task::spawn_blocking(move || -> rusqlite::Result<Option<f64>> {
|
let mut stmt =
|
||||||
let conn = open_db(&db_path)?;
|
conn.prepare("SELECT MIN(run_after) FROM tasks WHERE status='pending'")?;
|
||||||
let mut stmt = conn.prepare("SELECT MIN(run_after) FROM tasks WHERE status='pending'")?;
|
|
||||||
let mut rows = stmt.query([])?;
|
let mut rows = stmt.query([])?;
|
||||||
match rows.next()? {
|
match rows.next()? {
|
||||||
Some(row) => Ok(row.get::<_, Option<f64>>(0)?),
|
Some(row) => Ok(row.get::<_, Option<f64>>(0)?),
|
||||||
None => Ok(None),
|
None => Ok(None),
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
.expect("queue timing worker panicked")
|
match result {
|
||||||
.unwrap_or_else(|e| {
|
Ok(v) => v,
|
||||||
log::error!("queue timing query failed: {e}");
|
Err(e) => {
|
||||||
None
|
log::error!("queue timing query failed: {e}");
|
||||||
})
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn process(&self, row: LeasedRow) {
|
async fn process(&self, row: LeasedRow) {
|
||||||
@@ -336,33 +318,31 @@ impl QueueWorker {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn delete_row(&self, id: &str) {
|
async fn delete_row(&self, id: &str) {
|
||||||
let db_path = self.db_path.clone();
|
|
||||||
let id = id.to_string();
|
let id = id.to_string();
|
||||||
tokio::task::spawn_blocking(move || -> rusqlite::Result<()> {
|
let result = crate::db::with_conn(&self.db_path, move |conn| {
|
||||||
let conn = open_db(&db_path)?;
|
|
||||||
conn.execute("DELETE FROM tasks WHERE id = ?1", params![id])?;
|
conn.execute("DELETE FROM tasks WHERE id = ?1", params![id])?;
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
.expect("queue delete worker panicked")
|
if let Err(e) = result {
|
||||||
.unwrap_or_else(|e| log::error!("queue delete failed: {e}"));
|
log::error!("queue delete failed: {e}");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn reschedule(&self, id: &str, payload: Value, delay_seconds: f64, attempts: i32) {
|
async fn reschedule(&self, id: &str, payload: Value, delay_seconds: f64, attempts: i32) {
|
||||||
let db_path = self.db_path.clone();
|
|
||||||
let id = id.to_string();
|
let id = id.to_string();
|
||||||
let payload = payload.to_string();
|
let payload = payload.to_string();
|
||||||
tokio::task::spawn_blocking(move || -> rusqlite::Result<()> {
|
let result = crate::db::with_conn(&self.db_path, move |conn| {
|
||||||
let conn = open_db(&db_path)?;
|
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"UPDATE tasks SET payload=?1, run_after=?2, attempts=?3, status='pending', locked_until=0 WHERE id=?4",
|
"UPDATE tasks SET payload=?1, run_after=?2, attempts=?3, status='pending', locked_until=0 WHERE id=?4",
|
||||||
params![payload, now_f64() + delay_seconds, attempts, id],
|
params![payload, now_f64() + delay_seconds, attempts, id],
|
||||||
)?;
|
)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
.expect("queue reschedule worker panicked")
|
if let Err(e) = result {
|
||||||
.unwrap_or_else(|e| log::error!("queue reschedule failed: {e}"));
|
log::error!("queue reschedule failed: {e}");
|
||||||
|
}
|
||||||
self.notify.notify_waiters();
|
self.notify.notify_waiters();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+186
-107
@@ -5,20 +5,19 @@
|
|||||||
|
|
||||||
use crate::handlers::{CHAT_STORE, LINK_CACHE, TASK_QUEUE};
|
use crate::handlers::{CHAT_STORE, LINK_CACHE, TASK_QUEUE};
|
||||||
use crate::link_cache::{CachedMedia, CachedMediaKind, CachedPost};
|
use crate::link_cache::{CachedMedia, CachedMediaKind, CachedPost};
|
||||||
use crate::photo::{self, PhotoPrep, MAX_UPLOAD_BYTES};
|
use crate::photo::{self, MAX_UPLOAD_BYTES, PhotoPrep};
|
||||||
use crate::queue::QueueError;
|
use crate::queue::QueueError;
|
||||||
use crate::state::{EditMessage, unix_now};
|
use crate::state::{EditMessage, unix_now};
|
||||||
use rand::Rng;
|
use rand::Rng;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use tempfile::NamedTempFile;
|
|
||||||
use teloxide::prelude::*;
|
use teloxide::prelude::*;
|
||||||
use teloxide::types::{
|
use teloxide::types::{
|
||||||
ChatId, InlineKeyboardButton, InlineKeyboardMarkup, InputFile, InputMedia,
|
ChatId, InlineKeyboardButton, InlineKeyboardMarkup, InputFile, InputMedia, InputMediaAnimation,
|
||||||
InputMediaAnimation, InputMediaPhoto, InputMediaVideo, Message, MessageId, ParseMode,
|
InputMediaPhoto, InputMediaVideo, Message, MessageId, ParseMode, ReplyParameters,
|
||||||
ReplyParameters,
|
|
||||||
};
|
};
|
||||||
use teloxide::{ApiError, RequestError};
|
use teloxide::{ApiError, RequestError};
|
||||||
|
use tempfile::NamedTempFile;
|
||||||
use x_media::site::FetchError;
|
use x_media::site::FetchError;
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize, Clone, Debug)]
|
#[derive(Serialize, Deserialize, Clone, Debug)]
|
||||||
@@ -111,16 +110,18 @@ pub enum Task {
|
|||||||
impl Task {
|
impl Task {
|
||||||
fn cache_data(&self) -> Option<&CachedPost> {
|
fn cache_data(&self) -> Option<&CachedPost> {
|
||||||
match self {
|
match self {
|
||||||
Task::SendMediaSequence { cache_data, .. }
|
Task::SendMediaSequence { cache_data, .. } | Task::SendAnimation { cache_data, .. } => {
|
||||||
| Task::SendAnimation { cache_data, .. } => cache_data.as_ref(),
|
cache_data.as_ref()
|
||||||
|
}
|
||||||
Task::ForwardMessages { .. } => None,
|
Task::ForwardMessages { .. } => None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn source_url(&self) -> Option<&str> {
|
fn source_url(&self) -> Option<&str> {
|
||||||
match self {
|
match self {
|
||||||
Task::SendMediaSequence { source_url, .. }
|
Task::SendMediaSequence { source_url, .. } | Task::SendAnimation { source_url, .. } => {
|
||||||
| Task::SendAnimation { source_url, .. } => Some(source_url),
|
Some(source_url)
|
||||||
|
}
|
||||||
Task::ForwardMessages { .. } => None,
|
Task::ForwardMessages { .. } => None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -138,9 +139,10 @@ fn file_id_of_message(message: &Message, item: &MediaItemPayload) -> Option<Stri
|
|||||||
match item {
|
match item {
|
||||||
// `photo()` returns all sizes, smallest first — the largest carries
|
// `photo()` returns all sizes, smallest first — the largest carries
|
||||||
// the file id of the sent media.
|
// the file id of the sent media.
|
||||||
MediaItemPayload::Photo { .. } => {
|
MediaItemPayload::Photo { .. } => message
|
||||||
message.photo().and_then(|sizes| sizes.last()).map(|p| p.file.id.to_string())
|
.photo()
|
||||||
}
|
.and_then(|sizes| sizes.last())
|
||||||
|
.map(|p| p.file.id.to_string()),
|
||||||
MediaItemPayload::Video { .. } => message.video().map(|v| v.file.id.to_string()),
|
MediaItemPayload::Video { .. } => message.video().map(|v| v.file.id.to_string()),
|
||||||
MediaItemPayload::Animation { .. } => message.animation().map(|a| a.file.id.to_string()),
|
MediaItemPayload::Animation { .. } => message.animation().map(|a| a.file.id.to_string()),
|
||||||
}
|
}
|
||||||
@@ -214,7 +216,10 @@ pub const MAX_MEDIA_GROUP: usize = 9;
|
|||||||
|
|
||||||
/// Splits media into batches of at most [`MAX_MEDIA_GROUP`] items.
|
/// Splits media into batches of at most [`MAX_MEDIA_GROUP`] items.
|
||||||
pub fn chunk_media_items<T: Clone>(items: Vec<T>) -> Vec<Vec<T>> {
|
pub fn chunk_media_items<T: Clone>(items: Vec<T>) -> Vec<Vec<T>> {
|
||||||
items.chunks(MAX_MEDIA_GROUP).map(|chunk| chunk.to_vec()).collect()
|
items
|
||||||
|
.chunks(MAX_MEDIA_GROUP)
|
||||||
|
.map(|chunk| chunk.to_vec())
|
||||||
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Exponential backoff with jitter, capped at 30s.
|
/// Exponential backoff with jitter, capped at 30s.
|
||||||
@@ -250,31 +255,41 @@ pub fn is_size_error(e: &ApiError) -> bool {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
let description = e.to_string().to_lowercase();
|
let description = e.to_string().to_lowercase();
|
||||||
["too large", "too big"].iter().any(|marker| description.contains(marker))
|
["too large", "too big"]
|
||||||
|
.iter()
|
||||||
|
.any(|marker| description.contains(marker))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Task-free classification of a Telegram request error. The callers attach
|
/// Task-free classification of a Telegram request error. The callers attach
|
||||||
/// the (updated) task when building a [`SendError`].
|
/// the (updated) task when building a [`SendError`].
|
||||||
pub enum Classification {
|
pub enum Classification {
|
||||||
Retryable { delay_seconds: f64 },
|
Retryable {
|
||||||
Permanent { message: String },
|
delay_seconds: f64,
|
||||||
|
},
|
||||||
|
Permanent {
|
||||||
|
message: String,
|
||||||
|
},
|
||||||
/// Handled by the download fallback, not a queue retry.
|
/// Handled by the download fallback, not a queue retry.
|
||||||
MediaFetchFailure,
|
MediaFetchFailure,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn classify_request_error(e: &RequestError) -> Classification {
|
pub fn classify_request_error(e: &RequestError) -> Classification {
|
||||||
match e {
|
match e {
|
||||||
RequestError::RetryAfter(seconds) => {
|
RequestError::RetryAfter(seconds) => Classification::Retryable {
|
||||||
Classification::Retryable { delay_seconds: seconds.seconds() as f64 }
|
delay_seconds: seconds.seconds() as f64,
|
||||||
}
|
},
|
||||||
RequestError::Network(_) => Classification::Retryable {
|
RequestError::Network(_) => Classification::Retryable {
|
||||||
delay_seconds: retry_delay_seconds(0),
|
delay_seconds: retry_delay_seconds(0),
|
||||||
},
|
},
|
||||||
RequestError::Api(api) if is_media_fetch_failure(api) => Classification::MediaFetchFailure,
|
RequestError::Api(api) if is_media_fetch_failure(api) => Classification::MediaFetchFailure,
|
||||||
RequestError::Api(api) => Classification::Permanent { message: api.to_string() },
|
RequestError::Api(api) => Classification::Permanent {
|
||||||
|
message: api.to_string(),
|
||||||
|
},
|
||||||
RequestError::MigrateToChatId(_)
|
RequestError::MigrateToChatId(_)
|
||||||
| RequestError::InvalidJson { .. }
|
| RequestError::InvalidJson { .. }
|
||||||
| RequestError::Io(_) => Classification::Permanent { message: e.to_string() },
|
| RequestError::Io(_) => Classification::Permanent {
|
||||||
|
message: e.to_string(),
|
||||||
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -390,9 +405,9 @@ fn build_media_group(
|
|||||||
.map(|(i, item)| {
|
.map(|(i, item)| {
|
||||||
let item_caption = if i == 0 { caption } else { None };
|
let item_caption = if i == 0 { caption } else { None };
|
||||||
Ok(match item {
|
Ok(match item {
|
||||||
MediaItemPayload::Photo {
|
MediaItemPayload::Photo { has_spoiler, .. } => {
|
||||||
has_spoiler, ..
|
photo_media(item.input_file()?, item_caption, *has_spoiler)
|
||||||
} => photo_media(item.input_file()?, item_caption, *has_spoiler),
|
}
|
||||||
MediaItemPayload::Video {
|
MediaItemPayload::Video {
|
||||||
has_spoiler,
|
has_spoiler,
|
||||||
thumbnail,
|
thumbnail,
|
||||||
@@ -404,9 +419,9 @@ fn build_media_group(
|
|||||||
}
|
}
|
||||||
video
|
video
|
||||||
}
|
}
|
||||||
MediaItemPayload::Animation {
|
MediaItemPayload::Animation { has_spoiler, .. } => {
|
||||||
has_spoiler, ..
|
animation_media(item.input_file()?, item_caption, *has_spoiler)
|
||||||
} => animation_media(item.input_file()?, item_caption, *has_spoiler),
|
}
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
.collect()
|
.collect()
|
||||||
@@ -431,8 +446,12 @@ fn sniff_ext(bytes: &[u8]) -> &'static str {
|
|||||||
}
|
}
|
||||||
|
|
||||||
enum FallbackError {
|
enum FallbackError {
|
||||||
Retryable { delay_seconds: f64 },
|
Retryable {
|
||||||
Permanent { message: String },
|
delay_seconds: f64,
|
||||||
|
},
|
||||||
|
Permanent {
|
||||||
|
message: String,
|
||||||
|
},
|
||||||
/// The downloaded file exceeds the upload cap; the caller falls back to
|
/// The downloaded file exceeds the upload cap; the caller falls back to
|
||||||
/// the item's smaller URL.
|
/// the item's smaller URL.
|
||||||
MediaTooLarge,
|
MediaTooLarge,
|
||||||
@@ -468,9 +487,7 @@ async fn download_to_temp(item: &MediaItemPayload) -> Result<NamedTempFile, Fall
|
|||||||
};
|
};
|
||||||
// Photos are downloaded even over the cap so `prepare_photo` can
|
// Photos are downloaded even over the cap so `prepare_photo` can
|
||||||
// downscale / transcode them; only videos/animations short-circuit.
|
// downscale / transcode them; only videos/animations short-circuit.
|
||||||
if !matches!(item, MediaItemPayload::Photo { .. })
|
if !matches!(item, MediaItemPayload::Photo { .. }) && bytes.len() as u64 > MAX_UPLOAD_BYTES {
|
||||||
&& bytes.len() as u64 > MAX_UPLOAD_BYTES
|
|
||||||
{
|
|
||||||
return Err(FallbackError::MediaTooLarge);
|
return Err(FallbackError::MediaTooLarge);
|
||||||
}
|
}
|
||||||
let ext = sniff_ext(&bytes);
|
let ext = sniff_ext(&bytes);
|
||||||
@@ -628,14 +645,16 @@ async fn send_batch_via_upload(
|
|||||||
}
|
}
|
||||||
let result = bot
|
let result = bot
|
||||||
.send_media_group(ChatId(chat_id), items)
|
.send_media_group(ChatId(chat_id), items)
|
||||||
.reply_parameters(ReplyParameters::new(MessageId(reply_to as i32)).allow_sending_without_reply())
|
.reply_parameters(
|
||||||
|
ReplyParameters::new(MessageId(reply_to as i32)).allow_sending_without_reply(),
|
||||||
|
)
|
||||||
.await;
|
.await;
|
||||||
match result {
|
match result {
|
||||||
Ok(messages) => Ok(messages),
|
Ok(messages) => Ok(messages),
|
||||||
Err(e) => Err(match classify_request_error(&e) {
|
Err(e) => Err(match classify_request_error(&e) {
|
||||||
Classification::Retryable { delay_seconds } => FallbackError::Retryable {
|
Classification::Retryable { delay_seconds } => {
|
||||||
delay_seconds,
|
FallbackError::Retryable { delay_seconds }
|
||||||
},
|
}
|
||||||
Classification::Permanent { message } => FallbackError::Permanent { message },
|
Classification::Permanent { message } => FallbackError::Permanent { message },
|
||||||
Classification::MediaFetchFailure => FallbackError::Permanent {
|
Classification::MediaFetchFailure => FallbackError::Permanent {
|
||||||
message: "upload failed".into(),
|
message: "upload failed".into(),
|
||||||
@@ -702,7 +721,11 @@ pub async fn send_media_sequence(bot: &Bot, task: &Task) -> Result<Vec<i64>, Sen
|
|||||||
let fresh_send = *batch_index == 0 && sent.is_empty();
|
let fresh_send = *batch_index == 0 && sent.is_empty();
|
||||||
for idx in *batch_index..media_batches.len() {
|
for idx in *batch_index..media_batches.len() {
|
||||||
let batch = &media_batches[idx];
|
let batch = &media_batches[idx];
|
||||||
let caption = if idx == 0 { Some(caption.as_str()) } else { None };
|
let caption = if idx == 0 {
|
||||||
|
Some(caption.as_str())
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
let items = match build_media_group(batch, caption) {
|
let items = match build_media_group(batch, caption) {
|
||||||
Ok(items) => items,
|
Ok(items) => items,
|
||||||
Err(message) => {
|
Err(message) => {
|
||||||
@@ -714,7 +737,9 @@ pub async fn send_media_sequence(bot: &Bot, task: &Task) -> Result<Vec<i64>, Sen
|
|||||||
};
|
};
|
||||||
match bot
|
match bot
|
||||||
.send_media_group(ChatId(chat_id), items)
|
.send_media_group(ChatId(chat_id), items)
|
||||||
.reply_parameters(ReplyParameters::new(MessageId(reply_to as i32)).allow_sending_without_reply())
|
.reply_parameters(
|
||||||
|
ReplyParameters::new(MessageId(reply_to as i32)).allow_sending_without_reply(),
|
||||||
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(messages) => {
|
Ok(messages) => {
|
||||||
@@ -726,9 +751,7 @@ pub async fn send_media_sequence(bot: &Bot, task: &Task) -> Result<Vec<i64>, Sen
|
|||||||
collect_file_ids(&messages, batch, &mut cached_media);
|
collect_file_ids(&messages, batch, &mut cached_media);
|
||||||
sent.extend(messages.into_iter().map(|m| m.id.0 as i64));
|
sent.extend(messages.into_iter().map(|m| m.id.0 as i64));
|
||||||
}
|
}
|
||||||
Err(RequestError::Api(api))
|
Err(RequestError::Api(api)) if is_media_fetch_failure(&api) || is_size_error(&api) => {
|
||||||
if is_media_fetch_failure(&api) || is_size_error(&api) =>
|
|
||||||
{
|
|
||||||
log::info!(
|
log::info!(
|
||||||
"Telegram could not fetch media for batch {idx} ({}), downloading and reuploading",
|
"Telegram could not fetch media for batch {idx} ({}), downloading and reuploading",
|
||||||
batch.first().map(item_url).unwrap_or("?")
|
batch.first().map(item_url).unwrap_or("?")
|
||||||
@@ -779,7 +802,9 @@ async fn send_animation_inner(
|
|||||||
.send_animation(ChatId(chat_id), file)
|
.send_animation(ChatId(chat_id), file)
|
||||||
.caption(caption)
|
.caption(caption)
|
||||||
.parse_mode(ParseMode::Html)
|
.parse_mode(ParseMode::Html)
|
||||||
.reply_parameters(ReplyParameters::new(MessageId(reply_to as i32)).allow_sending_without_reply());
|
.reply_parameters(
|
||||||
|
ReplyParameters::new(MessageId(reply_to as i32)).allow_sending_without_reply(),
|
||||||
|
);
|
||||||
if spoiler {
|
if spoiler {
|
||||||
request = request.has_spoiler(true);
|
request = request.has_spoiler(true);
|
||||||
}
|
}
|
||||||
@@ -802,9 +827,7 @@ pub async fn send_animation(bot: &Bot, task: &Task) -> Result<Vec<i64>, SendErro
|
|||||||
let reply_to = *reply_to_message_id;
|
let reply_to = *reply_to_message_id;
|
||||||
let (media_url, has_spoiler) = match animation {
|
let (media_url, has_spoiler) = match animation {
|
||||||
MediaItemPayload::Animation {
|
MediaItemPayload::Animation {
|
||||||
media,
|
media, has_spoiler, ..
|
||||||
has_spoiler,
|
|
||||||
..
|
|
||||||
} => (media, *has_spoiler),
|
} => (media, *has_spoiler),
|
||||||
MediaItemPayload::Photo { .. } | MediaItemPayload::Video { .. } => {
|
MediaItemPayload::Photo { .. } | MediaItemPayload::Video { .. } => {
|
||||||
unreachable!("SendAnimation carries an Animation payload")
|
unreachable!("SendAnimation carries an Animation payload")
|
||||||
@@ -812,19 +835,20 @@ pub async fn send_animation(bot: &Bot, task: &Task) -> Result<Vec<i64>, SendErro
|
|||||||
};
|
};
|
||||||
let url_file = match input_file_for(media_url) {
|
let url_file = match input_file_for(media_url) {
|
||||||
Ok(file) => file,
|
Ok(file) => file,
|
||||||
Err(message) => return Err(SendError::Permanent { message, task: task.clone() }),
|
Err(message) => {
|
||||||
|
return Err(SendError::Permanent {
|
||||||
|
message,
|
||||||
|
task: task.clone(),
|
||||||
|
});
|
||||||
|
}
|
||||||
};
|
};
|
||||||
match send_animation_inner(bot, chat_id, reply_to, caption, has_spoiler, url_file)
|
match send_animation_inner(bot, chat_id, reply_to, caption, has_spoiler, url_file).await {
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(message) => {
|
Ok(message) => {
|
||||||
let id = message.id.0 as i64;
|
let id = message.id.0 as i64;
|
||||||
cache_animation_send(task, &message).await;
|
cache_animation_send(task, &message).await;
|
||||||
Ok(vec![id])
|
Ok(vec![id])
|
||||||
}
|
}
|
||||||
Err(RequestError::Api(api))
|
Err(RequestError::Api(api)) if is_media_fetch_failure(&api) || is_size_error(&api) => {
|
||||||
if is_media_fetch_failure(&api) || is_size_error(&api) =>
|
|
||||||
{
|
|
||||||
log::info!(
|
log::info!(
|
||||||
"Telegram could not fetch animation URL, downloading and reuploading: {}",
|
"Telegram could not fetch animation URL, downloading and reuploading: {}",
|
||||||
media_url
|
media_url
|
||||||
@@ -872,21 +896,24 @@ pub async fn send_animation(bot: &Bot, task: &Task) -> Result<Vec<i64>, SendErro
|
|||||||
Err(e) => Err(classify_to_send_error(&e, task.clone())),
|
Err(e) => Err(classify_to_send_error(&e, task.clone())),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(message) => {
|
Err(message) => Err(SendError::Permanent {
|
||||||
Err(SendError::Permanent { message, task: task.clone() })
|
message,
|
||||||
}
|
task: task.clone(),
|
||||||
|
}),
|
||||||
},
|
},
|
||||||
None => Err(SendError::Permanent {
|
None => Err(SendError::Permanent {
|
||||||
message: "media too large".into(),
|
message: "media too large".into(),
|
||||||
task: task.clone(),
|
task: task.clone(),
|
||||||
}),
|
}),
|
||||||
},
|
},
|
||||||
Err(FallbackError::Retryable { delay_seconds }) => {
|
Err(FallbackError::Retryable { delay_seconds }) => Err(SendError::Retryable {
|
||||||
Err(SendError::Retryable { delay_seconds, task: task.clone() })
|
delay_seconds,
|
||||||
}
|
task: task.clone(),
|
||||||
Err(FallbackError::Permanent { message }) => {
|
}),
|
||||||
Err(SendError::Permanent { message, task: task.clone() })
|
Err(FallbackError::Permanent { message }) => Err(SendError::Permanent {
|
||||||
}
|
message,
|
||||||
|
task: task.clone(),
|
||||||
|
}),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(e) => Err(classify_to_send_error(&e, task.clone())),
|
Err(e) => Err(classify_to_send_error(&e, task.clone())),
|
||||||
@@ -910,7 +937,11 @@ pub async fn forward_messages(bot: &Bot, task: &Task) -> Result<(), SendError> {
|
|||||||
.map(|id| MessageId(*id as i32))
|
.map(|id| MessageId(*id as i32))
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
match bot
|
match bot
|
||||||
.copy_messages(ChatId(*to_chat_id), ChatId(*from_chat_id), message_ids.clone())
|
.copy_messages(
|
||||||
|
ChatId(*to_chat_id),
|
||||||
|
ChatId(*from_chat_id),
|
||||||
|
message_ids.clone(),
|
||||||
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(_) => {
|
Ok(_) => {
|
||||||
@@ -944,12 +975,18 @@ pub fn build_edit_markup(templates: &HashMap<String, String>) -> InlineKeyboardM
|
|||||||
|
|
||||||
/// Notifies a chat about a dead-lettered task (skips when `notify_chat_id` is
|
/// Notifies a chat about a dead-lettered task (skips when `notify_chat_id` is
|
||||||
/// absent).
|
/// absent).
|
||||||
pub async fn notify_failure(bot: &Bot, chat_id: Option<i64>, message_id: Option<i64>, message: &str) {
|
pub async fn notify_failure(
|
||||||
|
bot: &Bot,
|
||||||
|
chat_id: Option<i64>,
|
||||||
|
message_id: Option<i64>,
|
||||||
|
message: &str,
|
||||||
|
) {
|
||||||
let Some(chat_id) = chat_id else { return };
|
let Some(chat_id) = chat_id else { return };
|
||||||
let mut request = bot.send_message(ChatId(chat_id), message);
|
let mut request = bot.send_message(ChatId(chat_id), message);
|
||||||
if let Some(message_id) = message_id {
|
if let Some(message_id) = message_id {
|
||||||
request = request
|
request = request.reply_parameters(
|
||||||
.reply_parameters(ReplyParameters::new(MessageId(message_id as i32)).allow_sending_without_reply());
|
ReplyParameters::new(MessageId(message_id as i32)).allow_sending_without_reply(),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
if let Err(e) = request.await {
|
if let Err(e) = request.await {
|
||||||
log::error!("failed to notify about failed task: {e}");
|
log::error!("failed to notify about failed task: {e}");
|
||||||
@@ -959,38 +996,45 @@ pub async fn notify_failure(bot: &Bot, chat_id: Option<i64>, message_id: Option<
|
|||||||
/// After a successful send: either open the edit-before-forward prompt or
|
/// After a successful send: either open the edit-before-forward prompt or
|
||||||
/// forward to the configured channel (with retry/queue handling).
|
/// forward to the configured channel (with retry/queue handling).
|
||||||
pub async fn post_send_actions(bot: &Bot, task: &Task, message_ids: Vec<i64>) {
|
pub async fn post_send_actions(bot: &Bot, task: &Task, message_ids: Vec<i64>) {
|
||||||
let (chat_id, reply_to, source_url, edit_before_forward, forward_channel_id, notify_chat_id, notify_message_id) =
|
let (
|
||||||
match task {
|
chat_id,
|
||||||
Task::SendMediaSequence {
|
reply_to,
|
||||||
chat_id,
|
source_url,
|
||||||
reply_to_message_id,
|
edit_before_forward,
|
||||||
source_url,
|
forward_channel_id,
|
||||||
edit_before_forward,
|
notify_chat_id,
|
||||||
forward_channel_id,
|
notify_message_id,
|
||||||
notify_chat_id,
|
) = match task {
|
||||||
notify_message_id,
|
Task::SendMediaSequence {
|
||||||
..
|
chat_id,
|
||||||
}
|
reply_to_message_id,
|
||||||
| Task::SendAnimation {
|
source_url,
|
||||||
chat_id,
|
edit_before_forward,
|
||||||
reply_to_message_id,
|
forward_channel_id,
|
||||||
source_url,
|
notify_chat_id,
|
||||||
edit_before_forward,
|
notify_message_id,
|
||||||
forward_channel_id,
|
..
|
||||||
notify_chat_id,
|
}
|
||||||
notify_message_id,
|
| Task::SendAnimation {
|
||||||
..
|
chat_id,
|
||||||
} => (
|
reply_to_message_id,
|
||||||
*chat_id,
|
source_url,
|
||||||
*reply_to_message_id,
|
edit_before_forward,
|
||||||
source_url.clone(),
|
forward_channel_id,
|
||||||
*edit_before_forward,
|
notify_chat_id,
|
||||||
*forward_channel_id,
|
notify_message_id,
|
||||||
*notify_chat_id,
|
..
|
||||||
*notify_message_id,
|
} => (
|
||||||
),
|
*chat_id,
|
||||||
Task::ForwardMessages { .. } => return,
|
*reply_to_message_id,
|
||||||
};
|
source_url.clone(),
|
||||||
|
*edit_before_forward,
|
||||||
|
*forward_channel_id,
|
||||||
|
*notify_chat_id,
|
||||||
|
*notify_message_id,
|
||||||
|
),
|
||||||
|
Task::ForwardMessages { .. } => return,
|
||||||
|
};
|
||||||
|
|
||||||
if edit_before_forward {
|
if edit_before_forward {
|
||||||
let mut chat_data = CHAT_STORE.get(chat_id).await;
|
let mut chat_data = CHAT_STORE.get(chat_id).await;
|
||||||
@@ -1027,7 +1071,10 @@ pub async fn post_send_actions(bot: &Bot, task: &Task, message_ids: Vec<i64>) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if let Some(channel_id) = forward_channel_id {
|
if let Some(channel_id) = forward_channel_id {
|
||||||
log::info!("forwarding {} message(s) to channel {channel_id}", message_ids.len());
|
log::info!(
|
||||||
|
"forwarding {} message(s) to channel {channel_id}",
|
||||||
|
message_ids.len()
|
||||||
|
);
|
||||||
let forward_task = Task::ForwardMessages {
|
let forward_task = Task::ForwardMessages {
|
||||||
from_chat_id: chat_id,
|
from_chat_id: chat_id,
|
||||||
to_chat_id: channel_id,
|
to_chat_id: channel_id,
|
||||||
@@ -1037,7 +1084,10 @@ pub async fn post_send_actions(bot: &Bot, task: &Task, message_ids: Vec<i64>) {
|
|||||||
};
|
};
|
||||||
match forward_messages(bot, &forward_task).await {
|
match forward_messages(bot, &forward_task).await {
|
||||||
Ok(()) => {}
|
Ok(()) => {}
|
||||||
Err(SendError::Retryable { delay_seconds, task }) => {
|
Err(SendError::Retryable {
|
||||||
|
delay_seconds,
|
||||||
|
task,
|
||||||
|
}) => {
|
||||||
let payload = serde_json::to_value(task).expect("task serializes");
|
let payload = serde_json::to_value(task).expect("task serializes");
|
||||||
let run_after = std::time::SystemTime::now()
|
let run_after = std::time::SystemTime::now()
|
||||||
.duration_since(std::time::UNIX_EPOCH)
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
@@ -1077,7 +1127,10 @@ pub async fn handle_task(payload: serde_json::Value) -> Result<(), QueueError> {
|
|||||||
Task::SendMediaSequence { .. } | Task::SendAnimation { .. } => {
|
Task::SendMediaSequence { .. } | Task::SendAnimation { .. } => {
|
||||||
let message_ids = match send_media_or_animation(&bot, &task).await {
|
let message_ids = match send_media_or_animation(&bot, &task).await {
|
||||||
Ok(ids) => ids,
|
Ok(ids) => ids,
|
||||||
Err(SendError::Retryable { delay_seconds, task }) => {
|
Err(SendError::Retryable {
|
||||||
|
delay_seconds,
|
||||||
|
task,
|
||||||
|
}) => {
|
||||||
return Err(QueueError::Retryable {
|
return Err(QueueError::Retryable {
|
||||||
delay_seconds,
|
delay_seconds,
|
||||||
payload: serde_json::to_value(task).expect("task serializes"),
|
payload: serde_json::to_value(task).expect("task serializes"),
|
||||||
@@ -1096,7 +1149,10 @@ pub async fn handle_task(payload: serde_json::Value) -> Result<(), QueueError> {
|
|||||||
}
|
}
|
||||||
Task::ForwardMessages { .. } => match forward_messages(&bot, &task).await {
|
Task::ForwardMessages { .. } => match forward_messages(&bot, &task).await {
|
||||||
Ok(()) => Ok(()),
|
Ok(()) => Ok(()),
|
||||||
Err(SendError::Retryable { delay_seconds, task }) => Err(QueueError::Retryable {
|
Err(SendError::Retryable {
|
||||||
|
delay_seconds,
|
||||||
|
task,
|
||||||
|
}) => Err(QueueError::Retryable {
|
||||||
delay_seconds,
|
delay_seconds,
|
||||||
payload: serde_json::to_value(task).expect("task serializes"),
|
payload: serde_json::to_value(task).expect("task serializes"),
|
||||||
}),
|
}),
|
||||||
@@ -1151,7 +1207,11 @@ mod tests {
|
|||||||
assert_eq!(chunk_media_items((0..10).collect()).len(), 2);
|
assert_eq!(chunk_media_items((0..10).collect()).len(), 2);
|
||||||
assert_eq!(chunk_media_items((0..25).collect()).len(), 3);
|
assert_eq!(chunk_media_items((0..25).collect()).len(), 3);
|
||||||
assert_eq!(chunk_media_items((0..25).collect())[2].len(), 7);
|
assert_eq!(chunk_media_items((0..25).collect())[2].len(), 7);
|
||||||
assert!(chunk_media_items((0..25).collect()).iter().all(|c| c.len() <= 9));
|
assert!(
|
||||||
|
chunk_media_items((0..25).collect())
|
||||||
|
.iter()
|
||||||
|
.all(|c| c.len() <= 9)
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -1175,7 +1235,10 @@ mod tests {
|
|||||||
let api = ApiError::Unknown(description.to_string());
|
let api = ApiError::Unknown(description.to_string());
|
||||||
assert!(is_media_fetch_failure(&api), "{description}");
|
assert!(is_media_fetch_failure(&api), "{description}");
|
||||||
}
|
}
|
||||||
for description in ["Bad Request: message is not modified", "Forbidden: bot was blocked by the user"] {
|
for description in [
|
||||||
|
"Bad Request: message is not modified",
|
||||||
|
"Forbidden: bot was blocked by the user",
|
||||||
|
] {
|
||||||
let api = ApiError::Unknown(description.to_string());
|
let api = ApiError::Unknown(description.to_string());
|
||||||
assert!(!is_media_fetch_failure(&api), "{description}");
|
assert!(!is_media_fetch_failure(&api), "{description}");
|
||||||
}
|
}
|
||||||
@@ -1196,7 +1259,10 @@ mod tests {
|
|||||||
assert!(is_size_error(&api), "{description}");
|
assert!(is_size_error(&api), "{description}");
|
||||||
}
|
}
|
||||||
// Unrelated errors must not match.
|
// Unrelated errors must not match.
|
||||||
for description in ["Bad Request: WEBPAGE_MEDIA_EMPTY", "Bad Request: message is not modified"] {
|
for description in [
|
||||||
|
"Bad Request: WEBPAGE_MEDIA_EMPTY",
|
||||||
|
"Bad Request: message is not modified",
|
||||||
|
] {
|
||||||
let api = ApiError::Unknown(description.to_string());
|
let api = ApiError::Unknown(description.to_string());
|
||||||
assert!(!is_size_error(&api), "{description}");
|
assert!(!is_size_error(&api), "{description}");
|
||||||
}
|
}
|
||||||
@@ -1205,9 +1271,16 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn media_item_payload_fallback_url_serde_default() {
|
fn media_item_payload_fallback_url_serde_default() {
|
||||||
// Old queued payloads without the field deserialize with None.
|
// Old queued payloads without the field deserialize with None.
|
||||||
let json = serde_json::json!({"kind": "photo", "media": "https://a/b.jpg", "has_spoiler": false});
|
let json =
|
||||||
|
serde_json::json!({"kind": "photo", "media": "https://a/b.jpg", "has_spoiler": false});
|
||||||
let photo: MediaItemPayload = serde_json::from_value(json).unwrap();
|
let photo: MediaItemPayload = serde_json::from_value(json).unwrap();
|
||||||
assert!(matches!(photo, MediaItemPayload::Photo { fallback_url: None, .. }));
|
assert!(matches!(
|
||||||
|
photo,
|
||||||
|
MediaItemPayload::Photo {
|
||||||
|
fallback_url: None,
|
||||||
|
..
|
||||||
|
}
|
||||||
|
));
|
||||||
assert_eq!(photo.fallback_url(), None);
|
assert_eq!(photo.fallback_url(), None);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1286,7 +1359,13 @@ mod tests {
|
|||||||
assert_eq!(sent_message_ids, vec![11, 12]);
|
assert_eq!(sent_message_ids, vec![11, 12]);
|
||||||
assert_eq!(forward_channel_id, Some(333));
|
assert_eq!(forward_channel_id, Some(333));
|
||||||
assert_eq!(media_batches.len(), 2);
|
assert_eq!(media_batches.len(), 2);
|
||||||
assert!(matches!(media_batches[0][0], MediaItemPayload::Photo { has_spoiler: true, .. }));
|
assert!(matches!(
|
||||||
|
media_batches[0][0],
|
||||||
|
MediaItemPayload::Photo {
|
||||||
|
has_spoiler: true,
|
||||||
|
..
|
||||||
|
}
|
||||||
|
));
|
||||||
}
|
}
|
||||||
other => panic!("expected SendMediaSequence, got {other:?}"),
|
other => panic!("expected SendMediaSequence, got {other:?}"),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
//! `data/task_queue.db`, shared with the task queue).
|
//! `data/task_queue.db`, shared with the task queue).
|
||||||
|
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
use rusqlite::{params, Connection};
|
use rusqlite::params;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
@@ -45,7 +45,9 @@ pub fn unix_now() -> i64 {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl ChatStore {
|
impl ChatStore {
|
||||||
/// Creates the parent directory and both tables (idempotent).
|
/// Creates the parent directory and the `chat_state` table (idempotent).
|
||||||
|
/// The shared `tasks` / `link_cache` tables are owned by `queue.rs` and
|
||||||
|
/// `link_cache.rs` respectively.
|
||||||
pub fn open(path: &str) -> rusqlite::Result<Self> {
|
pub fn open(path: &str) -> rusqlite::Result<Self> {
|
||||||
if let Some(parent) = Path::new(path).parent()
|
if let Some(parent) = Path::new(path).parent()
|
||||||
&& !parent.as_os_str().is_empty()
|
&& !parent.as_os_str().is_empty()
|
||||||
@@ -53,12 +55,9 @@ impl ChatStore {
|
|||||||
std::fs::create_dir_all(parent)
|
std::fs::create_dir_all(parent)
|
||||||
.map_err(|e| rusqlite::Error::ToSqlConversionFailure(Box::new(e)))?;
|
.map_err(|e| rusqlite::Error::ToSqlConversionFailure(Box::new(e)))?;
|
||||||
}
|
}
|
||||||
let conn = Connection::open(path)?;
|
let conn = crate::db::open_db(path)?;
|
||||||
conn.execute_batch(
|
conn.execute_batch(
|
||||||
"CREATE TABLE IF NOT EXISTS tasks (id TEXT PRIMARY KEY, payload TEXT NOT NULL, \
|
"CREATE TABLE IF NOT EXISTS chat_state (chat_id TEXT PRIMARY KEY, payload TEXT NOT NULL);",
|
||||||
run_after REAL NOT NULL, attempts INTEGER NOT NULL, status TEXT NOT NULL, \
|
|
||||||
locked_until REAL NOT NULL, created_at REAL NOT NULL); \
|
|
||||||
CREATE TABLE IF NOT EXISTS chat_state (chat_id TEXT PRIMARY KEY, payload TEXT NOT NULL);",
|
|
||||||
)?;
|
)?;
|
||||||
drop(conn);
|
drop(conn);
|
||||||
Ok(ChatStore {
|
Ok(ChatStore {
|
||||||
@@ -71,22 +70,19 @@ impl ChatStore {
|
|||||||
if let Some(data) = self.cache.lock().get(&chat_id) {
|
if let Some(data) = self.cache.lock().get(&chat_id) {
|
||||||
return data.clone();
|
return data.clone();
|
||||||
}
|
}
|
||||||
let db_path = self.db_path.clone();
|
let chat_key = chat_id.to_string();
|
||||||
let payload = tokio::task::spawn_blocking(move || -> rusqlite::Result<Option<String>> {
|
let payload = crate::db::with_conn(&self.db_path, move |conn| {
|
||||||
let conn = Connection::open(&db_path)?;
|
|
||||||
// Concurrent handler tasks (batch-forwards) may write chat_state
|
// Concurrent handler tasks (batch-forwards) may write chat_state
|
||||||
// while this read runs; without a busy timeout a write lock
|
// while this read runs; the shared busy timeout handles the
|
||||||
// collision fails the query immediately.
|
// write-lock collision instead of failing the query.
|
||||||
conn.busy_timeout(std::time::Duration::from_secs(5))?;
|
|
||||||
let mut stmt = conn.prepare("SELECT payload FROM chat_state WHERE chat_id = ?1")?;
|
let mut stmt = conn.prepare("SELECT payload FROM chat_state WHERE chat_id = ?1")?;
|
||||||
let mut rows = stmt.query(params![chat_id.to_string()])?;
|
let mut rows = stmt.query(params![chat_key])?;
|
||||||
match rows.next()? {
|
match rows.next()? {
|
||||||
Some(row) => Ok(Some(row.get(0)?)),
|
Some(row) => Ok(Some(row.get::<_, String>(0)?)),
|
||||||
None => Ok(None),
|
None => Ok(None),
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.expect("chat_state worker panicked")
|
|
||||||
.unwrap_or_else(|e| {
|
.unwrap_or_else(|e| {
|
||||||
log::error!("chat_state read failed: {e}");
|
log::error!("chat_state read failed: {e}");
|
||||||
None
|
None
|
||||||
@@ -101,19 +97,18 @@ impl ChatStore {
|
|||||||
pub async fn set(&self, chat_id: i64, data: &ChatData) {
|
pub async fn set(&self, chat_id: i64, data: &ChatData) {
|
||||||
self.cache.lock().insert(chat_id, data.clone());
|
self.cache.lock().insert(chat_id, data.clone());
|
||||||
let payload = serde_json::to_string(data).expect("chat state serializes");
|
let payload = serde_json::to_string(data).expect("chat state serializes");
|
||||||
let db_path = self.db_path.clone();
|
let chat_id = chat_id.to_string();
|
||||||
tokio::task::spawn_blocking(move || -> rusqlite::Result<()> {
|
let result = crate::db::with_conn(&self.db_path, move |conn| {
|
||||||
let conn = Connection::open(&db_path)?;
|
|
||||||
conn.busy_timeout(std::time::Duration::from_secs(5))?;
|
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"INSERT OR REPLACE INTO chat_state (chat_id, payload) VALUES (?1, ?2)",
|
"INSERT OR REPLACE INTO chat_state (chat_id, payload) VALUES (?1, ?2)",
|
||||||
params![chat_id.to_string(), payload],
|
params![chat_id, payload],
|
||||||
)?;
|
)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
.expect("chat_state worker panicked")
|
if let Err(e) = result {
|
||||||
.unwrap_or_else(|e| log::error!("chat_state write failed: {e}"));
|
log::error!("chat_state write failed: {e}");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Removes edit-before-forward records whose `created_at + ttl` is in the
|
/// Removes edit-before-forward records whose `created_at + ttl` is in the
|
||||||
@@ -149,7 +144,10 @@ impl ChatStore {
|
|||||||
self.set(chat_id, &data).await;
|
self.set(chat_id, &data).await;
|
||||||
}
|
}
|
||||||
if !removed.is_empty() {
|
if !removed.is_empty() {
|
||||||
log::info!("pruned {} expired edit-before-forward record(s)", removed.len());
|
log::info!(
|
||||||
|
"pruned {} expired edit-before-forward record(s)",
|
||||||
|
removed.len()
|
||||||
|
);
|
||||||
}
|
}
|
||||||
removed
|
removed
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,14 +11,14 @@ services:
|
|||||||
- html:/usr/share/nginx/html:ro
|
- html:/usr/share/nginx/html:ro
|
||||||
networks: [proxy]
|
networks: [proxy]
|
||||||
labels:
|
labels:
|
||||||
- 'com.github.nginx-proxy.nginx=true'
|
- 'com.github.nginx-proxy.nginx'
|
||||||
container_name: nginx-proxy
|
container_name: nginx-proxy
|
||||||
|
|
||||||
acme-companion:
|
acme-companion:
|
||||||
image: nginxproxy/acme-companion
|
image: nginxproxy/acme-companion
|
||||||
restart: always
|
restart: always
|
||||||
environment:
|
environment:
|
||||||
DEFAULT_EMAIL: 'admin@yoursfunny.top'
|
DEFAULT_EMAIL: ''
|
||||||
volumes:
|
volumes:
|
||||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||||
- certs:/etc/nginx/certs:rw
|
- certs:/etc/nginx/certs:rw
|
||||||
|
|||||||
Reference in New Issue
Block a user