mirror of
https://github.com/TheFunny/TelegramTwitterMediaBot.git
synced 2026-09-23 23:32:05 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 YoursFunny
|
ea72516d5c
|
||
|
YoursFunny
|
755330e585
|
||
|
YoursFunny
|
c40b074b3c
|
||
|
YoursFunny
|
9910da2914
|
||
|
YoursFunny
|
ebc0122264
|
||
|
YoursFunny
|
44cba8abe0
|
||
|
YoursFunny
|
99009aae9a
|
||
|
YoursFunny
|
72130b9023
|
||
|
YoursFunny
|
734cfc2eb3
|
||
|
YoursFunny
|
a8156697fa
|
||
|
YoursFunny
|
c093dfe5ac
|
||
|
YoursFunny
|
ee6f3e4a27
|
||
|
YoursFunny
|
16ed53fead
|
||
|
YoursFunny
|
1d9e3629c9
|
||
|
YoursFunny
|
b3d87b4f7d
|
||
|
YoursFunny
|
98c48b99c0
|
||
|
YoursFunny
|
f40639c799
|
||
|
YoursFunny
|
51cc079a85
|
||
|
YoursFunny
|
aa3083792a
|
||
|
YoursFunny
|
6849006ad7
|
||
|
YoursFunny
|
042a04ab6e
|
||
|
YoursFunny
|
9f28af4e6b
|
||
|
YoursFunny
|
d61dba5096
|
||
|
YoursFunny
|
f6df3e28cb
|
||
|
YoursFunny
|
deb1ef2428
|
||
|
YoursFunny
|
b5e5340edc
|
||
|
YoursFunny
|
425d1505cf
|
||
|
YoursFunny
|
6e40f55440
|
||
|
YoursFunny
|
7998114dc3
|
||
|
YoursFunny
|
9a96f78177
|
||
|
YoursFunny
|
b50f794d52
|
||
|
YoursFunny
|
d32fa969d6
|
Generated
+14
-364
@@ -99,28 +99,6 @@ version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
|
||||
|
||||
[[package]]
|
||||
name = "aws-lc-rs"
|
||||
version = "1.17.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00"
|
||||
dependencies = [
|
||||
"aws-lc-sys",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-lc-sys"
|
||||
version = "0.41.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cmake",
|
||||
"dunce",
|
||||
"fs_extra",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "axum"
|
||||
version = "0.8.9"
|
||||
@@ -264,12 +242,6 @@ version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
||||
|
||||
[[package]]
|
||||
name = "cfg_aliases"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
|
||||
|
||||
[[package]]
|
||||
name = "chrono"
|
||||
version = "0.4.44"
|
||||
@@ -292,15 +264,6 @@ dependencies = [
|
||||
"inout",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cmake"
|
||||
version = "0.1.58"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "colored"
|
||||
version = "3.1.1"
|
||||
@@ -310,16 +273,6 @@ dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "combine"
|
||||
version = "4.6.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "constant_time_eq"
|
||||
version = "0.3.1"
|
||||
@@ -530,12 +483,6 @@ dependencies = [
|
||||
"futures",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dunce"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
|
||||
|
||||
[[package]]
|
||||
name = "dyn-clone"
|
||||
version = "1.0.20"
|
||||
@@ -687,12 +634,6 @@ dependencies = [
|
||||
"percent-encoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fs_extra"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "futures"
|
||||
version = "0.3.32"
|
||||
@@ -798,10 +739,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"js-sys",
|
||||
"libc",
|
||||
"wasi",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1276,55 +1215,6 @@ version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||
|
||||
[[package]]
|
||||
name = "jni"
|
||||
version = "0.22.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"combine",
|
||||
"jni-macros",
|
||||
"jni-sys",
|
||||
"log",
|
||||
"simd_cesu8",
|
||||
"thiserror",
|
||||
"walkdir",
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jni-macros"
|
||||
version = "0.22.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"rustc_version",
|
||||
"simd_cesu8",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jni-sys"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2"
|
||||
dependencies = [
|
||||
"jni-sys-macros",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jni-sys-macros"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jobserver"
|
||||
version = "0.1.34"
|
||||
@@ -1409,12 +1299,6 @@ version = "0.4.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
|
||||
|
||||
[[package]]
|
||||
name = "lru-slab"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
|
||||
|
||||
[[package]]
|
||||
name = "lzma-rs"
|
||||
version = "0.3.0"
|
||||
@@ -1743,62 +1627,6 @@ dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quinn"
|
||||
version = "0.11.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"cfg_aliases",
|
||||
"pin-project-lite",
|
||||
"quinn-proto",
|
||||
"quinn-udp",
|
||||
"rustc-hash",
|
||||
"rustls",
|
||||
"socket2",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"web-time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quinn-proto"
|
||||
version = "0.11.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"bytes",
|
||||
"getrandom 0.3.4",
|
||||
"lru-slab",
|
||||
"rand 0.9.4",
|
||||
"ring",
|
||||
"rustc-hash",
|
||||
"rustls",
|
||||
"rustls-pki-types",
|
||||
"slab",
|
||||
"thiserror",
|
||||
"tinyvec",
|
||||
"tracing",
|
||||
"web-time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quinn-udp"
|
||||
version = "0.5.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd"
|
||||
dependencies = [
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
"once_cell",
|
||||
"socket2",
|
||||
"tracing",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.45"
|
||||
@@ -1827,18 +1655,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"rand_chacha 0.3.1",
|
||||
"rand_core 0.6.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.9.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea"
|
||||
dependencies = [
|
||||
"rand_chacha 0.9.0",
|
||||
"rand_core 0.9.5",
|
||||
"rand_chacha",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1848,17 +1666,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
|
||||
dependencies = [
|
||||
"ppv-lite86",
|
||||
"rand_core 0.6.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_chacha"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
|
||||
dependencies = [
|
||||
"ppv-lite86",
|
||||
"rand_core 0.9.5",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1870,15 +1678,6 @@ dependencies = [
|
||||
"getrandom 0.2.17",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"
|
||||
dependencies = [
|
||||
"getrandom 0.3.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rc-box"
|
||||
version = "1.3.0"
|
||||
@@ -1954,16 +1753,20 @@ checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"bytes",
|
||||
"encoding_rs",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"h2",
|
||||
"http",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
"hyper-rustls",
|
||||
"hyper-tls",
|
||||
"hyper-util",
|
||||
"js-sys",
|
||||
"log",
|
||||
"mime",
|
||||
"mime_guess",
|
||||
"native-tls",
|
||||
"percent-encoding",
|
||||
@@ -1986,47 +1789,6 @@ dependencies = [
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "reqwest"
|
||||
version = "0.13.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"bytes",
|
||||
"encoding_rs",
|
||||
"futures-core",
|
||||
"h2",
|
||||
"http",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
"hyper-rustls",
|
||||
"hyper-util",
|
||||
"js-sys",
|
||||
"log",
|
||||
"mime",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"quinn",
|
||||
"rustls",
|
||||
"rustls-pki-types",
|
||||
"rustls-platform-verifier",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_urlencoded",
|
||||
"sync_wrapper",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
"tower",
|
||||
"tower-http",
|
||||
"tower-service",
|
||||
"url",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rgb"
|
||||
version = "0.8.53"
|
||||
@@ -2064,21 +1826,6 @@ dependencies = [
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-hash"
|
||||
version = "2.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
|
||||
|
||||
[[package]]
|
||||
name = "rustc_version"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
|
||||
dependencies = [
|
||||
"semver",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "1.1.4"
|
||||
@@ -2098,7 +1845,6 @@ version = "0.23.40"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"once_cell",
|
||||
"rustls-pki-types",
|
||||
"rustls-webpki",
|
||||
@@ -2106,62 +1852,21 @@ dependencies = [
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-native-certs"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63"
|
||||
dependencies = [
|
||||
"openssl-probe",
|
||||
"rustls-pki-types",
|
||||
"schannel",
|
||||
"security-framework",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-pki-types"
|
||||
version = "1.14.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9"
|
||||
dependencies = [
|
||||
"web-time",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-platform-verifier"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0"
|
||||
dependencies = [
|
||||
"core-foundation 0.10.1",
|
||||
"core-foundation-sys",
|
||||
"jni",
|
||||
"log",
|
||||
"once_cell",
|
||||
"rustls",
|
||||
"rustls-native-certs",
|
||||
"rustls-platform-verifier-android",
|
||||
"rustls-webpki",
|
||||
"security-framework",
|
||||
"security-framework-sys",
|
||||
"webpki-root-certs",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-platform-verifier-android"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
|
||||
|
||||
[[package]]
|
||||
name = "rustls-webpki"
|
||||
version = "0.103.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"ring",
|
||||
"rustls-pki-types",
|
||||
"untrusted",
|
||||
@@ -2179,15 +1884,6 @@ version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
|
||||
|
||||
[[package]]
|
||||
name = "same-file"
|
||||
version = "1.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
|
||||
dependencies = [
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "schannel"
|
||||
version = "0.1.29"
|
||||
@@ -2387,22 +2083,6 @@ version = "0.3.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
|
||||
|
||||
[[package]]
|
||||
name = "simd_cesu8"
|
||||
version = "1.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33"
|
||||
dependencies = [
|
||||
"rustc_version",
|
||||
"simdutf8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "simdutf8"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
|
||||
|
||||
[[package]]
|
||||
name = "slab"
|
||||
version = "0.4.12"
|
||||
@@ -2536,7 +2216,7 @@ dependencies = [
|
||||
"log",
|
||||
"mime",
|
||||
"pin-project",
|
||||
"rand 0.8.6",
|
||||
"rand",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"teloxide-core",
|
||||
@@ -2567,7 +2247,7 @@ dependencies = [
|
||||
"once_cell",
|
||||
"pin-project",
|
||||
"rc-box",
|
||||
"reqwest 0.12.28",
|
||||
"reqwest",
|
||||
"rgb",
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -2914,16 +2594,6 @@ version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||
|
||||
[[package]]
|
||||
name = "walkdir"
|
||||
version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
|
||||
dependencies = [
|
||||
"same-file",
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "want"
|
||||
version = "0.3.1"
|
||||
@@ -3069,25 +2739,6 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "web-time"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "webpki-root-certs"
|
||||
version = "1.0.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c"
|
||||
dependencies = [
|
||||
"rustls-pki-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-util"
|
||||
version = "0.1.11"
|
||||
@@ -3351,15 +3002,15 @@ checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"
|
||||
|
||||
[[package]]
|
||||
name = "x-media"
|
||||
version = "1.0.8"
|
||||
version = "1.1.0"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"dotenv",
|
||||
"html-escape",
|
||||
"log",
|
||||
"rand 0.8.6",
|
||||
"rand",
|
||||
"regex",
|
||||
"reqwest 0.13.3",
|
||||
"reqwest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
@@ -3370,7 +3021,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "xmedia-bot"
|
||||
version = "1.0.8"
|
||||
version = "1.1.0"
|
||||
dependencies = [
|
||||
"dotenv",
|
||||
"fast_image_resize",
|
||||
@@ -3380,8 +3031,7 @@ dependencies = [
|
||||
"parking_lot",
|
||||
"png",
|
||||
"pretty_env_logger",
|
||||
"rand 0.8.6",
|
||||
"regex",
|
||||
"rand",
|
||||
"rusqlite",
|
||||
"serde",
|
||||
"serde_json",
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
[workspace]
|
||||
members = ["crates/x-media", "crates/xmedia-bot"]
|
||||
resolver = "3"
|
||||
|
||||
# Smaller production binary; debug symbols are not shipped anyway.
|
||||
[profile.release]
|
||||
strip = true
|
||||
|
||||
+14
-9
@@ -11,6 +11,11 @@ ARG APP_NAME=telegram-twitter-media-bot
|
||||
# runners. `/redirect/latest/` floats to the newest release build; each build
|
||||
# also ships a .sha256. Swap `amd64` for `arm64` when building arm64 images.
|
||||
ARG FFMPEG_URL=https://ffmpeg.martin-riedl.de/redirect/latest/linux/amd64/release/ffmpeg.zip
|
||||
# Optional sha256 of ffmpeg.zip (pinned releases only): set to verify the
|
||||
# download. The mirror publishes .sha256 sidecars next to pinned builds, e.g.
|
||||
# https://ffmpeg.martin-riedl.de/download/linux/amd64/<id>_9.0/ffmpeg.zip.sha256
|
||||
# (the /redirect/latest/ URL itself has no sidecar — pin the effective URL).
|
||||
ARG FFMPEG_SHA256=
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
@@ -30,19 +35,20 @@ RUN mkdir -p crates/x-media/src crates/xmedia-bot/src \
|
||||
# root. `unzip -t` verifies the archive before extraction so a bad
|
||||
# download fails loudly here instead of a cryptic later error.
|
||||
RUN wget -q -O /tmp/ffmpeg.zip "$FFMPEG_URL" \
|
||||
&& if [ -n "$FFMPEG_SHA256" ]; then echo "$FFMPEG_SHA256 /tmp/ffmpeg.zip" | sha256sum -c -; fi \
|
||||
&& unzip -tq /tmp/ffmpeg.zip \
|
||||
&& unzip -q /tmp/ffmpeg.zip -d /usr/local/bin \
|
||||
&& chmod +x /usr/local/bin/ffmpeg \
|
||||
&& rm /tmp/ffmpeg.zip \
|
||||
&& /usr/local/bin/ffmpeg -version >/dev/null
|
||||
|
||||
# 3. Real sources last: only our crates recompile on source changes. The
|
||||
# COPY preserves host mtimes, which predate the stub artifacts from step 1;
|
||||
# cargo's mtime-based freshness check would otherwise treat the stub build
|
||||
# as up-to-date and never compile the real sources. `touch` forces cargo to
|
||||
# see the real files as newer.
|
||||
# 3. Real sources last: only our crates recompile on source changes.
|
||||
# `cargo clean -p` drops the two crates' artifacts while keeping the
|
||||
# compiled dependency layer, forcing a deterministic rebuild of the real
|
||||
# sources. (The previous `touch`-mtimes hack silently shipped the stub
|
||||
# binary when host files carried future timestamps.)
|
||||
COPY crates/ ./crates/
|
||||
RUN find crates -type f -name '*.rs' -exec touch {} + \
|
||||
RUN cargo clean -p xmedia-bot -p x-media \
|
||||
&& cargo build --release -p xmedia-bot
|
||||
|
||||
# ---------- runtime stage ----------
|
||||
@@ -56,10 +62,9 @@ LABEL org.opencontainers.image.title="${APP_NAME}"
|
||||
|
||||
# Everything is copied in — no apt in the runtime stage. Privilege dropping is
|
||||
# done by docker-entrypoint.sh with setpriv (util-linux, already in
|
||||
# bookworm-slim), so no gosu needed.
|
||||
# bookworm-slim), so no gosu needed. (libssl3/libcrypto are already in
|
||||
# bookworm-slim; only ca-certificates and ffmpeg need copying.)
|
||||
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
COPY --from=builder /usr/lib/x86_64-linux-gnu/libssl.so.3* /usr/lib/x86_64-linux-gnu/
|
||||
COPY --from=builder /usr/lib/x86_64-linux-gnu/libcrypto.so.3* /usr/lib/x86_64-linux-gnu/
|
||||
COPY --from=builder /usr/local/bin/ffmpeg /usr/local/bin/ffmpeg
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
[package]
|
||||
name = "x-media"
|
||||
version = "1.0.8"
|
||||
version = "1.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
reqwest = { version = "0.13", features = ["json", "query", "form"] }
|
||||
reqwest = { version = "0.12", features = ["json"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
regex = "1.12"
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
use super::model;
|
||||
use crate::media::Media;
|
||||
use crate::site::{FetchError, Fetched};
|
||||
use html_escape::encode_text;
|
||||
use html_escape::{encode_double_quoted_attribute, encode_text};
|
||||
use regex::Regex;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub static PATTERN: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"bsky\.app/profile/([\w.\-:]+)/post/([\w.\-~]+)").unwrap());
|
||||
pub static PATTERN: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r"^(?:https?://)?bsky\.app/profile/([\w.\-:]+)/post/([\w.\-~]+)").unwrap()
|
||||
});
|
||||
|
||||
pub fn enabled() -> bool {
|
||||
true
|
||||
@@ -22,7 +23,161 @@ pub async fn fetch_from_url(url: &str) -> Result<Fetched, FetchError> {
|
||||
.get(2)
|
||||
.map(|m| m.as_str())
|
||||
.ok_or(FetchError::NotFound)?;
|
||||
Ok(fetch(handle, rkey).await?.into())
|
||||
let post = fetch(handle, rkey).await?;
|
||||
let mut fetched: Fetched = post.into();
|
||||
// bsky video embeds expose only an HLS playlist URL, which Telegram
|
||||
// cannot fetch; remux it to a single MP4 (mirrors the pixiv ugoira
|
||||
// encode path — the temp file stays alive via `_keep_alive`). On any
|
||||
// failure the video item is dropped and the post degrades to its text.
|
||||
let mut media = Vec::with_capacity(fetched.media.len());
|
||||
for item in fetched.media {
|
||||
let is_hls = matches!(&item, Media::Video { url, .. }
|
||||
if url.contains("playlist") || url.ends_with(".m3u8"));
|
||||
if !is_hls {
|
||||
media.push(item);
|
||||
continue;
|
||||
}
|
||||
let url = item.url().to_string();
|
||||
match resolve_bsky_video(&url).await {
|
||||
Ok(Some((mp4_path, keep_alive))) => {
|
||||
let thumbnail_url = match &item {
|
||||
Media::Video { thumbnail_url, .. } => thumbnail_url.clone(),
|
||||
_ => String::new(),
|
||||
};
|
||||
media.push(Media::Video {
|
||||
title: None,
|
||||
url: mp4_path.to_string_lossy().into_owned(),
|
||||
thumbnail_url,
|
||||
});
|
||||
fetched._keep_alive = Some(keep_alive);
|
||||
}
|
||||
Ok(None) => log::warn!("bsky video remux unavailable for {url}"),
|
||||
Err(e) => log::warn!("bsky video remux failed for {url}: {e}"),
|
||||
}
|
||||
}
|
||||
fetched.media = media;
|
||||
Ok(fetched)
|
||||
}
|
||||
|
||||
/// Downloads an HLS playlist (master or media) and remuxes its segments to a
|
||||
/// single MP4 via ffmpeg. Returns the MP4 path plus the temp dir that must
|
||||
/// stay alive until the file is uploaded. `Ok(None)` when ffmpeg is missing.
|
||||
///
|
||||
/// Verified live (2026-08): bsky master playlists carry `#EXT-X-STREAM-INF`
|
||||
/// variant lines (e.g. `720p/video.m3u8?session_id=…`), and the media
|
||||
/// playlists are VOD MPEG-TS segments (`videoN.ts?…`) without EXT-X-MAP, so
|
||||
/// a plain `-f concat -c copy` remux is valid.
|
||||
async fn resolve_bsky_video(
|
||||
playlist_url: &str,
|
||||
) -> Result<Option<(std::path::PathBuf, tempfile::TempDir)>, String> {
|
||||
if !crate::site::ffmpeg_available() {
|
||||
crate::site::log_once_ffmpeg_missing();
|
||||
return Ok(None);
|
||||
}
|
||||
let master = crate::site::download_media_limited(playlist_url, 1_048_576)
|
||||
.await
|
||||
.map_err(|e| format!("bsky video master playlist: {e}"))?;
|
||||
let master = String::from_utf8_lossy(&master);
|
||||
|
||||
// Master playlist: pick the variant with the highest declared bandwidth.
|
||||
let playlist_url = if master.contains("#EXT-X-STREAM-INF") {
|
||||
let mut best: Option<(u64, String)> = None;
|
||||
let mut lines = master.lines();
|
||||
while let Some(line) = lines.next() {
|
||||
if !line.starts_with("#EXT-X-STREAM-INF") {
|
||||
continue;
|
||||
}
|
||||
let bandwidth = line
|
||||
.split_once("BANDWIDTH=")
|
||||
.and_then(|(_, rest)| rest.split(|c: char| !c.is_ascii_digit()).next())
|
||||
.and_then(|n| n.parse::<u64>().ok())
|
||||
.unwrap_or(0);
|
||||
if let Some(uri) = lines.next().filter(|u| !u.starts_with('#'))
|
||||
&& bandwidth >= best.as_ref().map(|(b, _)| *b).unwrap_or(0)
|
||||
{
|
||||
best = Some((bandwidth, uri.to_string()));
|
||||
}
|
||||
}
|
||||
let Some((_, uri)) = best else {
|
||||
return Err("bsky video master playlist has no variants".to_string());
|
||||
};
|
||||
url::Url::parse(playlist_url)
|
||||
.and_then(|base| base.join(&uri))
|
||||
.map_err(|e| format!("bsky video variant URL: {e}"))?
|
||||
.to_string()
|
||||
} else {
|
||||
playlist_url.to_string()
|
||||
};
|
||||
|
||||
let variant = crate::site::download_media_limited(&playlist_url, 1_048_576)
|
||||
.await
|
||||
.map_err(|e| format!("bsky video media playlist: {e}"))?;
|
||||
let variant = String::from_utf8_lossy(&variant);
|
||||
// Segment URIs: non-#, non-empty lines, resolved relative to the playlist.
|
||||
let base = url::Url::parse(&playlist_url).map_err(|e| format!("bsky playlist URL: {e}"))?;
|
||||
let segments: Vec<String> = variant
|
||||
.lines()
|
||||
.map(str::trim)
|
||||
.filter(|l| !l.is_empty() && !l.starts_with('#'))
|
||||
.map(|l| base.join(l).map(|u| u.to_string()))
|
||||
.collect::<Result<_, _>>()
|
||||
.map_err(|e| format!("bsky segment URL: {e}"))?;
|
||||
if segments.is_empty() {
|
||||
return Err("bsky video playlist has no segments".to_string());
|
||||
}
|
||||
if segments.len() > 500 {
|
||||
return Err("bsky video has too many segments".to_string());
|
||||
}
|
||||
|
||||
let frames_dir = tempfile::tempdir().map_err(|e| e.to_string())?;
|
||||
let out_dir = tempfile::tempdir().map_err(|e| e.to_string())?;
|
||||
let mut total: u64 = 0;
|
||||
let mut list = String::new();
|
||||
for (i, seg) in segments.iter().enumerate() {
|
||||
let bytes = crate::site::download_media_limited(seg, 20 * 1024 * 1024)
|
||||
.await
|
||||
.map_err(|e| format!("bsky segment {i}: {e}"))?;
|
||||
total += bytes.len() as u64;
|
||||
if total > 256 * 1024 * 1024 {
|
||||
return Err("bsky video exceeds total size cap".to_string());
|
||||
}
|
||||
let path = frames_dir.path().join(format!("seg_{i:04}.ts"));
|
||||
std::fs::write(&path, &bytes).map_err(|e| e.to_string())?;
|
||||
list.push_str(&format!("file '{}'\n", path.to_string_lossy()));
|
||||
}
|
||||
let list_path = frames_dir.path().join("list.txt");
|
||||
std::fs::write(&list_path, &list).map_err(|e| e.to_string())?;
|
||||
|
||||
let output = out_dir.path().join("video.mp4");
|
||||
let list_str = list_path.to_string_lossy().into_owned();
|
||||
let output_str = output.to_string_lossy().into_owned();
|
||||
let status = tokio::task::spawn_blocking(move || {
|
||||
std::process::Command::new("ffmpeg")
|
||||
.args([
|
||||
"-y",
|
||||
"-f",
|
||||
"concat",
|
||||
"-safe",
|
||||
"0",
|
||||
"-i",
|
||||
&list_str,
|
||||
"-c",
|
||||
"copy",
|
||||
"-movflags",
|
||||
"+faststart",
|
||||
&output_str,
|
||||
])
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.status()
|
||||
})
|
||||
.await
|
||||
.map_err(|e| format!("bsky remux worker panicked: {e}"))?;
|
||||
match status {
|
||||
Ok(s) if s.success() => Ok(Some((output, out_dir))),
|
||||
Ok(s) => Err(format!("ffmpeg exited with {s}")),
|
||||
Err(e) => Err(format!("ffmpeg spawn failed: {e}")),
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetches a post thread by handle or DID (`at://` URIs work for both).
|
||||
@@ -35,8 +190,16 @@ pub async fn fetch(handle: &str, rkey: &str) -> Result<Post, FetchError> {
|
||||
])
|
||||
.send()
|
||||
.await?;
|
||||
// 404/410 = gone (permanent); 429/5xx = transient and retried by fetch.
|
||||
let status = response.status();
|
||||
if !status.is_success() {
|
||||
return match status.as_u16() {
|
||||
404 | 410 => Err(FetchError::NotFound),
|
||||
_ => Err(FetchError::Transient(format!("bsky status {status}"))),
|
||||
};
|
||||
}
|
||||
let text = response.text().await?;
|
||||
Ok(Post::from_json(&text, rkey.to_string())?)
|
||||
Post::from_json(&text, rkey.to_string())
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -61,8 +224,8 @@ impl Post {
|
||||
pub fn caption(&self) -> String {
|
||||
format!(
|
||||
"{url}\n<a href=\"{author_url}\">{author}</a>: {text}",
|
||||
url = self.url(),
|
||||
author_url = self.author_url(),
|
||||
url = encode_double_quoted_attribute(&self.url()),
|
||||
author_url = encode_double_quoted_attribute(&self.author_url()),
|
||||
author = encode_text(&self.author),
|
||||
text = encode_text(&self.text),
|
||||
)
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
|
||||
use std::fmt;
|
||||
use std::sync::LazyLock;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
pub mod bsky;
|
||||
@@ -147,6 +148,10 @@ pub enum FetchError {
|
||||
/// The post exists but its content is withheld (twitter NSFW /
|
||||
/// age-restricted tweets come back as an empty `{}` from syndication).
|
||||
Sensitive,
|
||||
/// A download exceeded the caller's size cap (see [`download_media_limited`]).
|
||||
TooLarge,
|
||||
/// A transient server-side failure (429 / 5xx); [`fetch`] retries these.
|
||||
Transient(String),
|
||||
}
|
||||
|
||||
impl fmt::Display for FetchError {
|
||||
@@ -158,6 +163,8 @@ impl fmt::Display for FetchError {
|
||||
FetchError::NotFound => write!(f, "not found"),
|
||||
FetchError::Blocked => write!(f, "blocked"),
|
||||
FetchError::Sensitive => write!(f, "content withheld (sensitive)"),
|
||||
FetchError::TooLarge => write!(f, "media too large"),
|
||||
FetchError::Transient(message) => write!(f, "transient: {message}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -169,6 +176,8 @@ impl std::error::Error for FetchError {
|
||||
FetchError::Json(e) => Some(e),
|
||||
FetchError::Pixiv(e) => Some(e),
|
||||
FetchError::NotFound | FetchError::Blocked | FetchError::Sensitive => None,
|
||||
FetchError::TooLarge => None,
|
||||
FetchError::Transient(_) => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -194,7 +203,22 @@ impl From<PixivError> for FetchError {
|
||||
/// Shared HTTP client (browser User-Agent) for twitter/bsky fetches and
|
||||
/// [`download_media`].
|
||||
pub(crate) static CLIENT: LazyLock<reqwest::Client> = LazyLock::new(|| {
|
||||
let builder = reqwest::Client::builder().user_agent("Mozilla/5.0");
|
||||
let mut builder = reqwest::Client::builder()
|
||||
.user_agent("Mozilla/5.0")
|
||||
// reqwest has no total timeout by default; a stalled connection
|
||||
// would otherwise pin a fetch/handler forever.
|
||||
.timeout(Duration::from_secs(30))
|
||||
.connect_timeout(Duration::from_secs(10));
|
||||
// Route site fetches through the same proxy the Bot API uses, so a
|
||||
// network that needs TELOXIDE_PROXY (e.g. behind the GFW) does not
|
||||
// leave site fetches dead while the bot itself works.
|
||||
if let Some(proxy) = std::env::var("TELOXIDE_PROXY")
|
||||
.ok()
|
||||
.filter(|s| !s.is_empty())
|
||||
&& let Ok(p) = reqwest::Proxy::all(&proxy)
|
||||
{
|
||||
builder = builder.proxy(p);
|
||||
}
|
||||
// Each `#[tokio::test]` runs on its own runtime; the connection pool is
|
||||
// bound to the runtime that created it, so cross-runtime reuse of idle
|
||||
// connections fails with DispatchGone. In test builds every request uses
|
||||
@@ -204,13 +228,38 @@ pub(crate) static CLIENT: LazyLock<reqwest::Client> = LazyLock::new(|| {
|
||||
builder.build().expect("failed to build HTTP client")
|
||||
});
|
||||
|
||||
/// Whether a usable `ffmpeg` binary is on PATH (probed once). Shared by the
|
||||
/// pixiv ugoira encoder and the bsky HLS remuxer.
|
||||
static FFMPEG_AVAILABLE: LazyLock<bool> = LazyLock::new(|| {
|
||||
std::process::Command::new("ffmpeg")
|
||||
.arg("-version")
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.status()
|
||||
.map(|s| s.success())
|
||||
.unwrap_or(false)
|
||||
});
|
||||
|
||||
static FFMPEG_MISSING_LOGGED: AtomicBool = AtomicBool::new(false);
|
||||
|
||||
pub(crate) fn ffmpeg_available() -> bool {
|
||||
*FFMPEG_AVAILABLE
|
||||
}
|
||||
|
||||
pub(crate) fn log_once_ffmpeg_missing() {
|
||||
if !FFMPEG_MISSING_LOGGED.swap(true, Ordering::Relaxed) {
|
||||
log::warn!("ffmpeg not found; ugoira and bsky video posts stay unsupported");
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetches a post from its URL. Returns `Ok(None)` when no site pattern
|
||||
/// matches (unsupported links are silently ignored by the bot).
|
||||
///
|
||||
/// Transient network failures are retried: 3 total attempts with 1s then 2s
|
||||
/// delays. Non-Http errors (Json/NotFound/Blocked/Pixiv) are not retried.
|
||||
/// delays. Retried classes: bare HTTP errors, [`FetchError::Transient`]
|
||||
/// (429/5xx from any site), and pixiv errors (its network failures arrive
|
||||
/// wrapped as `PixivError`). Non-retried: Json/NotFound/Blocked/Sensitive.
|
||||
pub async fn fetch(url: &str) -> Result<Option<Fetched>, FetchError> {
|
||||
let mut last_http_error = None;
|
||||
for attempt in 0..3u32 {
|
||||
match fetch_once(url).await {
|
||||
Ok(Some(fetched)) => {
|
||||
@@ -222,18 +271,17 @@ pub async fn fetch(url: &str) -> Result<Option<Fetched>, FetchError> {
|
||||
return Ok(Some(fetched));
|
||||
}
|
||||
Ok(None) => return Ok(None),
|
||||
Err(FetchError::Http(e)) => {
|
||||
last_http_error = Some(e);
|
||||
Err(e @ (FetchError::Http(_) | FetchError::Transient(_) | FetchError::Pixiv(_))) => {
|
||||
if attempt < 2 {
|
||||
tokio::time::sleep(Duration::from_secs(1 << attempt)).await;
|
||||
} else {
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
Err(other) => return Err(other),
|
||||
}
|
||||
}
|
||||
Err(FetchError::Http(
|
||||
last_http_error.expect("retry loop always ran 3 attempts"),
|
||||
))
|
||||
unreachable!("retry loop always returns")
|
||||
}
|
||||
|
||||
async fn fetch_once(url: &str) -> Result<Option<Fetched>, FetchError> {
|
||||
@@ -262,18 +310,39 @@ pub async fn media_size(url: &str) -> Result<Option<u64>, FetchError> {
|
||||
if lower.contains("pximg.net") {
|
||||
request = request.header("Referer", "https://www.pixiv.net/");
|
||||
}
|
||||
let response = request.send().await?;
|
||||
let response = request.send().await?.error_for_status()?;
|
||||
Ok(response.content_length())
|
||||
}
|
||||
|
||||
pub async fn download_media(url: &str) -> Result<bytes::Bytes, FetchError> {
|
||||
/// Downloads a media file with a hard size cap: the body is streamed and the
|
||||
/// download aborts with [`FetchError::TooLarge`] the moment the cap is
|
||||
/// crossed (or when a declared Content-Length already exceeds it). Keeps the
|
||||
/// bot from buffering arbitrarily large bodies into memory.
|
||||
pub async fn download_media_limited(url: &str, max_bytes: u64) -> Result<bytes::Bytes, FetchError> {
|
||||
let mut request = CLIENT.get(url);
|
||||
let lower = url.to_ascii_lowercase();
|
||||
if lower.contains("pximg.net") {
|
||||
request = request.header("Referer", "https://www.pixiv.net/");
|
||||
}
|
||||
let response = request.send().await?;
|
||||
Ok(response.bytes().await?)
|
||||
let response = request.send().await?.error_for_status()?;
|
||||
if let Some(len) = response.content_length()
|
||||
&& len > max_bytes
|
||||
{
|
||||
return Err(FetchError::TooLarge);
|
||||
}
|
||||
let mut response = response;
|
||||
let mut buf = Vec::new();
|
||||
while let Some(chunk) = response.chunk().await? {
|
||||
buf.extend_from_slice(&chunk);
|
||||
if buf.len() as u64 > max_bytes {
|
||||
return Err(FetchError::TooLarge);
|
||||
}
|
||||
}
|
||||
Ok(bytes::Bytes::from(buf))
|
||||
}
|
||||
|
||||
pub async fn download_media(url: &str) -> Result<bytes::Bytes, FetchError> {
|
||||
download_media_limited(url, u64::MAX).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -136,6 +136,9 @@ impl PixivAPI {
|
||||
.bearer_auth(access_token)
|
||||
.send()
|
||||
.await?;
|
||||
if !response.status().is_success() {
|
||||
return Err(PixivError::Api(format!("status {}", response.status())));
|
||||
}
|
||||
let json: serde_json::Value = serde_json::from_str(&response.text().await?)?;
|
||||
if json.get("error").is_some() {
|
||||
let message = json
|
||||
@@ -186,6 +189,9 @@ impl PixivAPI {
|
||||
.bearer_auth(access_token)
|
||||
.send()
|
||||
.await?;
|
||||
if !response.status().is_success() {
|
||||
return Err(PixivError::Api(format!("status {}", response.status())));
|
||||
}
|
||||
let json: serde_json::Value = serde_json::from_str(&response.text().await?)?;
|
||||
if json.get("error").is_some() {
|
||||
let message = json
|
||||
@@ -207,8 +213,8 @@ impl PixivAPI {
|
||||
&self,
|
||||
illust_id: u64,
|
||||
) -> Result<Option<(String, tempfile::TempDir)>, PixivError> {
|
||||
if !ffmpeg_available() {
|
||||
log_once_ffmpeg_missing();
|
||||
if !crate::site::ffmpeg_available() {
|
||||
crate::site::log_once_ffmpeg_missing();
|
||||
return Ok(None);
|
||||
}
|
||||
let metadata = self.ugoira_metadata(illust_id).await?;
|
||||
@@ -222,7 +228,7 @@ impl PixivAPI {
|
||||
let Some(zip_url) = zip_url else {
|
||||
return Ok(None);
|
||||
};
|
||||
let zip_bytes = crate::site::download_media(&zip_url)
|
||||
let zip_bytes = crate::site::download_media_limited(&zip_url, 512 * 1024 * 1024)
|
||||
.await
|
||||
.map_err(|e| match e {
|
||||
FetchError::Http(e) => PixivError::Http(e),
|
||||
@@ -238,23 +244,48 @@ impl PixivAPI {
|
||||
// frames are uniformly jpg or png per artwork.
|
||||
let mut archive = zip::ZipArchive::new(Cursor::new(zip_bytes))
|
||||
.map_err(|e| format!("unzip: {e}"))?;
|
||||
// pixiv ugoira frames are uniformly jpg or png per artwork; take
|
||||
// the extension from the first entry.
|
||||
let extension = if archive.len() > 0 {
|
||||
let first_name = archive
|
||||
.by_index(0)
|
||||
.map_err(|e| e.to_string())?
|
||||
.name()
|
||||
.to_string();
|
||||
first_name.rsplit('.').next().unwrap_or("jpg").to_string()
|
||||
if archive.is_empty() {
|
||||
return Err("empty frame zip".to_string());
|
||||
}
|
||||
// Uniform jpg or png per artwork; sniff the first entry's
|
||||
// magic bytes instead of trusting its filename.
|
||||
let first = archive.by_index(0).map_err(|e| e.to_string())?;
|
||||
let mut first_bytes = Vec::new();
|
||||
first
|
||||
.take(64 * 1024 * 1024 + 1)
|
||||
.read_to_end(&mut first_bytes)
|
||||
.map_err(|e| e.to_string())?;
|
||||
if first_bytes.len() > 64 * 1024 * 1024 {
|
||||
return Err("frame exceeds size cap".to_string());
|
||||
}
|
||||
let extension = if first_bytes.starts_with(&[0xFF, 0xD8]) {
|
||||
"jpg"
|
||||
} else if first_bytes.starts_with(b"\x89PNG") {
|
||||
"png"
|
||||
} else {
|
||||
"jpg".to_string()
|
||||
"jpg"
|
||||
};
|
||||
let mut count = 0usize;
|
||||
for i in 0..archive.len() {
|
||||
let mut entry = archive.by_index(i).map_err(|e| e.to_string())?;
|
||||
{
|
||||
let path = frames_dir
|
||||
.path()
|
||||
.join(format!("img_{count:05}.{extension}"));
|
||||
std::fs::write(&path, &first_bytes).map_err(|e| e.to_string())?;
|
||||
count += 1;
|
||||
}
|
||||
for i in 1..archive.len() {
|
||||
let entry = archive.by_index(i).map_err(|e| e.to_string())?;
|
||||
if entry.size() > 64 * 1024 * 1024 {
|
||||
return Err(format!("frame {i} exceeds size cap"));
|
||||
}
|
||||
let mut bytes = Vec::new();
|
||||
entry.read_to_end(&mut bytes).map_err(|e| e.to_string())?;
|
||||
entry
|
||||
.take(64 * 1024 * 1024 + 1)
|
||||
.read_to_end(&mut bytes)
|
||||
.map_err(|e| e.to_string())?;
|
||||
if bytes.len() > 64 * 1024 * 1024 {
|
||||
return Err(format!("frame {i} exceeds size cap"));
|
||||
}
|
||||
let path = frames_dir
|
||||
.path()
|
||||
.join(format!("img_{count:05}.{extension}"));
|
||||
@@ -304,7 +335,10 @@ impl PixivAPI {
|
||||
Ok((output.to_string_lossy().into_owned(), out_dir))
|
||||
})
|
||||
.await
|
||||
.expect("ugoira encode worker panicked");
|
||||
.map_err(|e| {
|
||||
log::error!("ugoira encode worker panicked for {illust_id}: {e}");
|
||||
PixivError::Api(format!("ugoira worker failed: {e}"))
|
||||
})?;
|
||||
match result {
|
||||
Ok(pair) => Ok(Some(pair)),
|
||||
Err(message) => {
|
||||
@@ -315,28 +349,6 @@ impl PixivAPI {
|
||||
}
|
||||
}
|
||||
|
||||
static FFMPEG_AVAILABLE: LazyLock<bool> = LazyLock::new(|| {
|
||||
std::process::Command::new("ffmpeg")
|
||||
.arg("-version")
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.status()
|
||||
.map(|s| s.success())
|
||||
.unwrap_or(false)
|
||||
});
|
||||
|
||||
static FFMPEG_MISSING_LOGGED: AtomicBool = AtomicBool::new(false);
|
||||
|
||||
fn ffmpeg_available() -> bool {
|
||||
*FFMPEG_AVAILABLE
|
||||
}
|
||||
|
||||
fn log_once_ffmpeg_missing() {
|
||||
if !FFMPEG_MISSING_LOGGED.swap(true, Ordering::Relaxed) {
|
||||
log::warn!("ffmpeg not found; pixiv ugoira posts stay unsupported");
|
||||
}
|
||||
}
|
||||
|
||||
/// pixiv3-rs replacement: `None` when `PIXIV_REFRESH_TOKEN` is unset.
|
||||
static PIXIV_CLIENT: LazyLock<Option<PixivAPI>> =
|
||||
LazyLock::new(|| env::var("PIXIV_REFRESH_TOKEN").ok().map(PixivAPI::new));
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
use super::model::{IllustrationModel, TypeModel};
|
||||
use crate::media::Media;
|
||||
use crate::site::{FetchError, Fetched};
|
||||
use html_escape::encode_text;
|
||||
use html_escape::{encode_double_quoted_attribute, encode_text};
|
||||
use regex::Regex;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub static PATTERN: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r"(?:www\.)?pixiv\.net/(?:en/)?(?:(?:i|artworks)/|member_illust\.php\?(?:mode=[a-z_]*&)?illust_id=)(\d+)").unwrap()
|
||||
Regex::new(r"^(?:https?://)?(?:www\.)?pixiv\.net/(?:en/)?(?:(?:i|artworks)/|member_illust\.php\?(?:mode=[a-z_]*&)?illust_id=)(\d+)").unwrap()
|
||||
});
|
||||
|
||||
pub fn enabled() -> bool {
|
||||
@@ -48,9 +48,9 @@ impl Illustration {
|
||||
pub fn caption(&self) -> String {
|
||||
format!(
|
||||
"<a href=\"{url}\">{title}</a> / <a href=\"{author_url}\">{author}</a>\n{tags}",
|
||||
url = self.url(),
|
||||
url = encode_double_quoted_attribute(&self.url()),
|
||||
title = encode_text(&self.title),
|
||||
author_url = self.author_url(),
|
||||
author_url = encode_double_quoted_attribute(&self.author_url()),
|
||||
author = encode_text(&self.author),
|
||||
tags = encode_text(
|
||||
&self
|
||||
|
||||
@@ -126,9 +126,16 @@ pub async fn fetch(id: &str) -> Result<Tweet, FetchError> {
|
||||
.header("referer", "https://x.com/")
|
||||
.send()
|
||||
.await?;
|
||||
if !response.status().is_success() {
|
||||
log::warn!("twitter auth fetch {id}: HTTP {}", response.status());
|
||||
return Err(FetchError::NotFound);
|
||||
// 404/410 = gone (permanent); 429/5xx = transient and retried by fetch.
|
||||
let status = response.status();
|
||||
if !status.is_success() {
|
||||
log::warn!("twitter auth fetch {id}: HTTP {status}");
|
||||
return match status.as_u16() {
|
||||
404 | 410 => Err(FetchError::NotFound),
|
||||
_ => Err(FetchError::Transient(format!(
|
||||
"twitter auth status {status}"
|
||||
))),
|
||||
};
|
||||
}
|
||||
let text = response.text().await?;
|
||||
let json: Value = serde_json::from_str(&text)?;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use super::model;
|
||||
use crate::media::Media;
|
||||
use crate::site::{FetchError, Fetched};
|
||||
use html_escape::encode_text;
|
||||
use html_escape::{encode_double_quoted_attribute, encode_text};
|
||||
use regex::Regex;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
@@ -48,7 +48,9 @@ pub async fn fetch_from_url(url: &str) -> Result<Fetched, FetchError> {
|
||||
fn empty_fetched(url: &str) -> Fetched {
|
||||
Fetched {
|
||||
source_url: url.to_string(),
|
||||
caption: url.to_string(),
|
||||
// The raw user-supplied URL goes into an HTML caption; escape it so
|
||||
// crafted links cannot break the parse (Telegram 400).
|
||||
caption: encode_text(url).into_owned(),
|
||||
title: String::new(),
|
||||
media: vec![],
|
||||
sensitive: true,
|
||||
@@ -68,8 +70,13 @@ pub async fn fetch(id: &str) -> Result<Tweet, FetchError> {
|
||||
))
|
||||
.send()
|
||||
.await?;
|
||||
if !response.status().is_success() {
|
||||
return Err(FetchError::NotFound);
|
||||
// 404/410 = gone (permanent); 429/5xx = transient and retried by fetch.
|
||||
let status = response.status();
|
||||
if !status.is_success() {
|
||||
return match status.as_u16() {
|
||||
404 | 410 => Err(FetchError::NotFound),
|
||||
_ => Err(FetchError::Transient(format!("twitter status {status}"))),
|
||||
};
|
||||
}
|
||||
let text = response.text().await?;
|
||||
// Deleted tweets answer with {"errors": [...]} instead of a tweet.
|
||||
@@ -146,8 +153,8 @@ impl Tweet {
|
||||
pub fn caption(&self) -> String {
|
||||
format!(
|
||||
"{url}\n<a href=\"{author_url}\">{author}</a>: {text}",
|
||||
url = self.url(),
|
||||
author_url = self.author_url(),
|
||||
url = encode_double_quoted_attribute(&self.url()),
|
||||
author_url = encode_double_quoted_attribute(&self.author_url()),
|
||||
author = encode_text(&self.author),
|
||||
text = encode_text(&self.text),
|
||||
)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "xmedia-bot"
|
||||
version = "1.0.8"
|
||||
version = "1.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
@@ -12,7 +12,6 @@ log = "0.4"
|
||||
pretty_env_logger = "0.5"
|
||||
dotenv = "0.15"
|
||||
url = "2.5.2"
|
||||
regex = "1.12"
|
||||
html-escape = "0.2"
|
||||
rusqlite = { version = "0.32", features = ["bundled"] }
|
||||
rand = "0.8"
|
||||
|
||||
@@ -16,9 +16,22 @@ use std::time::Duration;
|
||||
pub fn open_db(path: &str) -> rusqlite::Result<Connection> {
|
||||
let conn = Connection::open(path)?;
|
||||
conn.busy_timeout(Duration::from_secs(5))?;
|
||||
// WAL lets readers run alongside writer leases instead of blocking on
|
||||
// the rollback journal; the mode persists in the DB header, so the
|
||||
// idempotent pragma here and in ensure_schema only needs to win once.
|
||||
conn.pragma_update(None, "journal_mode", "WAL")?;
|
||||
Ok(conn)
|
||||
}
|
||||
|
||||
/// Unix timestamp in fractional seconds. Shared by the queue, chat store and
|
||||
/// link cache (previously four private copies).
|
||||
pub fn now_f64() -> f64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_secs_f64())
|
||||
.unwrap_or(0.0)
|
||||
}
|
||||
|
||||
/// Runs `f` against a fresh connection on a blocking thread, returning the
|
||||
/// closure's result. Owns the `spawn_blocking` + `expect` ceremony shared by
|
||||
/// every table access; the caller maps errors to its own log line.
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use crate::config::Config;
|
||||
use crate::db::now_f64;
|
||||
use crate::link_cache::{CachedMediaKind, CachedPost, LinkCache};
|
||||
use crate::queue::PersistentTaskQueue;
|
||||
use crate::send::{self, MediaItemPayload, Task};
|
||||
@@ -13,9 +14,51 @@ use teloxide::types::{
|
||||
MessageEntityKind, MessageId, ParseMode, Recipient, ReplyParameters,
|
||||
};
|
||||
use teloxide::utils::command::BotCommands;
|
||||
use tokio::sync::Semaphore;
|
||||
use x_media::media::Media;
|
||||
|
||||
/// One URL job: bot handle + the message + the extracted URL.
|
||||
type UrlJob = (Bot, Message, String);
|
||||
/// Bounded channel of URL jobs drained by [`start_url_workers`]. The bound
|
||||
/// caps both queued memory and shutdown backlog; a full channel applies
|
||||
/// backpressure to the per-chat handler instead of spawning unbounded tasks.
|
||||
static URL_JOBS: LazyLock<parking_lot::Mutex<Option<tokio::sync::mpsc::Sender<UrlJob>>>> =
|
||||
LazyLock::new(|| parking_lot::Mutex::new(None));
|
||||
/// Set by main's shutdown sequence; workers stop pulling new jobs.
|
||||
static URL_STOP: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
|
||||
|
||||
/// Worker count draining URL jobs; keeps the old 8-permit concurrency cap
|
||||
/// while bounding how many jobs can be queued at all.
|
||||
const URL_WORKERS: usize = 8;
|
||||
|
||||
/// Starts the URL job workers (called once from main after the queue starts).
|
||||
/// teloxide dispatches updates to a per-chat worker that handles them
|
||||
/// sequentially, so a batch-forward of many messages would otherwise be
|
||||
/// processed one at a time (fetch + send each, roughly a second per
|
||||
/// message); the workers add throughput, and FIFO order preserves per-message
|
||||
/// URL order.
|
||||
pub async fn start_url_workers() {
|
||||
let (tx, rx) = tokio::sync::mpsc::channel::<UrlJob>(256);
|
||||
*URL_JOBS.lock() = Some(tx);
|
||||
let rx = std::sync::Arc::new(tokio::sync::Mutex::new(rx));
|
||||
for _ in 0..URL_WORKERS {
|
||||
let rx = std::sync::Arc::clone(&rx);
|
||||
tokio::spawn(async move {
|
||||
while !URL_STOP.load(std::sync::atomic::Ordering::Relaxed) {
|
||||
let job = rx.lock().await.recv().await;
|
||||
match job {
|
||||
Some((bot, message, url)) => url_media(bot, &message, &url).await,
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// Stops URL workers (drains up to the 256 queued jobs, then exits).
|
||||
pub fn stop_url_workers() {
|
||||
URL_STOP.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub static CHAT_STORE: LazyLock<ChatStore> =
|
||||
LazyLock::new(|| ChatStore::open("data/task_queue.db").expect("failed to open chat store"));
|
||||
pub static TASK_QUEUE: LazyLock<PersistentTaskQueue> =
|
||||
@@ -24,14 +67,6 @@ pub static LINK_CACHE: LazyLock<LinkCache> =
|
||||
LazyLock::new(|| LinkCache::open("data/task_queue.db"));
|
||||
pub static CONFIG: LazyLock<Config> = LazyLock::new(Config::load);
|
||||
|
||||
/// Cap on concurrent per-URL processing. teloxide dispatches updates to a
|
||||
/// per-chat worker that handles them sequentially, so a batch-forward of many
|
||||
/// messages would otherwise be processed one at a time (fetch + send each,
|
||||
/// roughly a second per message). Moving the work into spawned tasks trades
|
||||
/// per-chat reply ordering for throughput; the semaphore bounds how many run
|
||||
/// at once so a big burst cannot hammer Telegram's rate limits.
|
||||
static URL_TASKS: LazyLock<Semaphore> = LazyLock::new(|| Semaphore::new(8));
|
||||
|
||||
#[derive(BotCommands, Clone)]
|
||||
#[command(
|
||||
rename_rule = "snake_case",
|
||||
@@ -76,13 +111,6 @@ where
|
||||
.await
|
||||
}
|
||||
|
||||
fn now_f64() -> f64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_secs_f64())
|
||||
.unwrap_or(0.0)
|
||||
}
|
||||
|
||||
/// Extracts URL and text-link entities (text + caption), deduped in order.
|
||||
pub fn extract_urls(message: &Message) -> Vec<String> {
|
||||
let mut urls = Vec::new();
|
||||
@@ -101,7 +129,10 @@ pub fn extract_urls(message: &Message) -> Vec<String> {
|
||||
}
|
||||
}
|
||||
let mut seen = HashSet::new();
|
||||
urls.retain(|url| seen.insert(url.clone()));
|
||||
// Dedup by the normalized post id so variant URLs of the same post
|
||||
// (/status/1 vs /status/1/photo/1) are sent once; unsupported URLs fall
|
||||
// back to exact-string dedup.
|
||||
urls.retain(|url| seen.insert(x_media::site::cache_key(url).unwrap_or_else(|| url.clone())));
|
||||
urls
|
||||
}
|
||||
|
||||
@@ -124,7 +155,7 @@ async fn edit_message_handler(bot: &Bot, message: &Message) -> bool {
|
||||
};
|
||||
let link = format!(
|
||||
"<a href=\"{0}\">{1}</a>",
|
||||
edit.url,
|
||||
html_escape::encode_double_quoted_attribute(&edit.url),
|
||||
html_escape::encode_text(text)
|
||||
);
|
||||
let new_text = if edit.template.is_empty() {
|
||||
@@ -190,19 +221,31 @@ async fn set_forward_channel_handler(
|
||||
return Err(SetForwardChannelError::NotChannel);
|
||||
}
|
||||
let channel_id = chat.id.0;
|
||||
// The sender must be a channel administrator. Compare against the
|
||||
// sender's user id, NOT the chat id (they only coincide in private
|
||||
// chats, so the old check broke group usage).
|
||||
let Some(sender) = message.from.as_ref() else {
|
||||
return Err(SetForwardChannelError::NotAdmin);
|
||||
};
|
||||
match bot.get_chat_administrators(channel.clone()).await {
|
||||
Err(e) => {
|
||||
log::error!("Failed to get channel administrators {}: {}", channel, e);
|
||||
return Err(SetForwardChannelError::NotBotAdmin(e));
|
||||
}
|
||||
Ok(admins) => {
|
||||
if !admins.iter().any(|admin| admin.user.id == message.chat.id) {
|
||||
if !admins.iter().any(|admin| admin.user.id == sender.id) {
|
||||
return Err(SetForwardChannelError::NotAdmin);
|
||||
}
|
||||
let bot_id = bot.get_me().await.expect("Failed get bot id").user.id;
|
||||
if let Some(bot_admin) = admins.iter().find(|admin| admin.user.id == bot_id)
|
||||
&& !bot_admin.can_post_messages()
|
||||
{
|
||||
// The bot itself must be an admin that can post; a missing
|
||||
// bot entry must not pass silently (copy would fail later).
|
||||
let bot_id = match bot.get_me().await {
|
||||
Ok(me) => me.user.id,
|
||||
Err(e) => return Err(SetForwardChannelError::NotBotAdmin(e)),
|
||||
};
|
||||
let bot_ok = admins
|
||||
.iter()
|
||||
.any(|admin| admin.user.id == bot_id && admin.can_post_messages());
|
||||
if !bot_ok {
|
||||
return Err(SetForwardChannelError::NotBotCanPost);
|
||||
}
|
||||
}
|
||||
@@ -226,9 +269,11 @@ async fn execute_command(
|
||||
Command::SetForwardChannel(channel) => {
|
||||
let result = match set_forward_channel_handler(bot, message, channel).await {
|
||||
Ok(channel_id) => {
|
||||
let mut chat_data = CHAT_STORE.get(message.chat.id.0).await;
|
||||
chat_data.forward_channel_id = Some(channel_id);
|
||||
CHAT_STORE.set(message.chat.id.0, &chat_data).await;
|
||||
CHAT_STORE
|
||||
.update(message.chat.id.0, |data| {
|
||||
data.forward_channel_id = Some(channel_id);
|
||||
})
|
||||
.await;
|
||||
"Add successfully.".to_string()
|
||||
}
|
||||
Err(SetForwardChannelError::EmptyParameter) => {
|
||||
@@ -252,31 +297,34 @@ async fn execute_command(
|
||||
}
|
||||
Command::RemoveForwardChannel => {
|
||||
let chat_id = message.chat.id.0;
|
||||
let mut chat_data = CHAT_STORE.get(chat_id).await;
|
||||
let text = if chat_data.forward_channel_id.is_some() {
|
||||
chat_data.forward_channel_id = None;
|
||||
CHAT_STORE.set(chat_id, &chat_data).await;
|
||||
"Remove successfully.".to_string()
|
||||
} else {
|
||||
"No channel to remove.".to_string()
|
||||
};
|
||||
let text = CHAT_STORE
|
||||
.update(chat_id, |data| {
|
||||
if data.forward_channel_id.is_some() {
|
||||
data.forward_channel_id = None;
|
||||
"Remove successfully.".to_string()
|
||||
} else {
|
||||
"No channel to remove.".to_string()
|
||||
}
|
||||
})
|
||||
.await;
|
||||
reply(bot.clone(), message.clone(), text).await?;
|
||||
}
|
||||
Command::EditBeforeForward => {
|
||||
let chat_id = message.chat.id.0;
|
||||
let mut chat_data = CHAT_STORE.get(chat_id).await;
|
||||
let text = if chat_data.forward_channel_id.is_none() {
|
||||
"Please enable forward channel first.".to_string()
|
||||
} else if chat_data.edit_before_forward {
|
||||
chat_data.edit_before_forward = false;
|
||||
chat_data.edit_message.clear();
|
||||
CHAT_STORE.set(chat_id, &chat_data).await;
|
||||
"Disable edit before forward.".to_string()
|
||||
} else {
|
||||
chat_data.edit_before_forward = true;
|
||||
CHAT_STORE.set(chat_id, &chat_data).await;
|
||||
"Enable edit before forward.".to_string()
|
||||
};
|
||||
let text = CHAT_STORE
|
||||
.update(chat_id, |data| {
|
||||
if data.forward_channel_id.is_none() {
|
||||
"Please enable forward channel first.".to_string()
|
||||
} else if data.edit_before_forward {
|
||||
data.edit_before_forward = false;
|
||||
data.edit_message.clear();
|
||||
"Disable edit before forward.".to_string()
|
||||
} else {
|
||||
data.edit_before_forward = true;
|
||||
"Enable edit before forward.".to_string()
|
||||
}
|
||||
})
|
||||
.await;
|
||||
reply(bot.clone(), message.clone(), text).await?;
|
||||
}
|
||||
Command::SetTemplate(name) => {
|
||||
@@ -290,11 +338,14 @@ async fn execute_command(
|
||||
} else if name.is_empty() {
|
||||
"Please provide a name for the template.".to_string()
|
||||
} else {
|
||||
let mut chat_data = CHAT_STORE.get(chat_id).await;
|
||||
chat_data
|
||||
.template
|
||||
.insert(name, html_escape::encode_text(reply_text).into_owned());
|
||||
CHAT_STORE.set(chat_id, &chat_data).await;
|
||||
CHAT_STORE
|
||||
.update(chat_id, |data| {
|
||||
data.template.insert(
|
||||
name,
|
||||
html_escape::encode_text(reply_text).into_owned(),
|
||||
);
|
||||
})
|
||||
.await;
|
||||
"Template set.".to_string()
|
||||
}
|
||||
}
|
||||
@@ -332,9 +383,11 @@ async fn execute_command(
|
||||
.await?;
|
||||
return Ok(());
|
||||
}
|
||||
let mut chat_data = CHAT_STORE.get(chat_id).await;
|
||||
chat_data.message_format.insert(site.to_string(), format);
|
||||
CHAT_STORE.set(chat_id, &chat_data).await;
|
||||
CHAT_STORE
|
||||
.update(chat_id, |data| {
|
||||
data.message_format.insert(site.to_string(), format);
|
||||
})
|
||||
.await;
|
||||
reply(bot.clone(), message.clone(), "Format set.").await?;
|
||||
}
|
||||
Command::ClearCache(arg) => {
|
||||
@@ -388,9 +441,18 @@ async fn execute_command(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `""` for one, `"ies"` for anything else — "1 entry" / "2 entries".
|
||||
/// `"y"` for one, `"ies"` for anything else — "1 entry" / "2 entries".
|
||||
fn plural(n: usize) -> &'static str {
|
||||
if n == 1 { "" } else { "ies" }
|
||||
if n == 1 { "y" } else { "ies" }
|
||||
}
|
||||
|
||||
/// Registers the bot's command list with Telegram so clients show it in the
|
||||
/// `/` menu (Bot API `setMyCommands`).
|
||||
pub async fn register_commands(bot: &Bot) -> Result<(), RequestError> {
|
||||
let commands = Command::bot_commands();
|
||||
bot.set_my_commands(commands.clone()).await?;
|
||||
log::info!("registered {} commands", commands.len());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// For locally produced media (encoded ugoira MP4) the thumbnail URL is a
|
||||
@@ -664,7 +726,10 @@ pub async fn message_handler(bot: Bot, message: Message) -> Result<(), RequestEr
|
||||
.unwrap_or_else(|| "unknown".to_string());
|
||||
let text_preview = message
|
||||
.text()
|
||||
.map(|t| if t.len() > 120 { &t[..120] } else { t })
|
||||
.map(|t| {
|
||||
let end = t.floor_char_boundary(120.min(t.len()));
|
||||
&t[..end]
|
||||
})
|
||||
.unwrap_or("<no text>");
|
||||
log::info!(
|
||||
"message from {sender} in {} (private={is_private}): {text_preview}",
|
||||
@@ -687,13 +752,13 @@ pub async fn message_handler(bot: Bot, message: Message) -> Result<(), RequestEr
|
||||
log::info!("extracted {} URL(s): {urls:?}", urls.len());
|
||||
}
|
||||
for url in urls {
|
||||
let bot = bot.clone();
|
||||
let message = message.clone();
|
||||
tokio::spawn(async move {
|
||||
// Held for the whole task; the semaphore is never closed.
|
||||
let _permit = URL_TASKS.acquire().await.expect("URL semaphore closed");
|
||||
url_media(bot, &message, &url).await;
|
||||
});
|
||||
// Clone out of the lock: the parking_lot guard is !Send and must
|
||||
// not be held across the await below.
|
||||
let Some(tx) = URL_JOBS.lock().clone() else {
|
||||
log::warn!("url workers not started; dropping link");
|
||||
break;
|
||||
};
|
||||
let _ = tx.send((bot.clone(), message.clone(), url)).await;
|
||||
}
|
||||
}
|
||||
respond(())
|
||||
@@ -703,6 +768,12 @@ pub async fn inline_query_handler(bot: Bot, query: InlineQuery) -> Result<(), Re
|
||||
if query.query.is_empty() {
|
||||
return respond(());
|
||||
}
|
||||
// Telegram fires an inline query on every keystroke; only run a fetch
|
||||
// (3 attempts!) for something that is actually a supported post URL, so
|
||||
// typing does not hammer the source sites.
|
||||
if x_media::site::cache_key(&query.query).is_none() {
|
||||
return respond(());
|
||||
}
|
||||
log::info!("inline query: {}", query.query);
|
||||
match x_media::site::fetch(&query.query).await {
|
||||
Ok(Some(fetched)) => {
|
||||
@@ -769,7 +840,7 @@ pub async fn callback_query_handler(bot: Bot, query: CallbackQuery) -> Result<()
|
||||
let chat_id = message.chat().id.0;
|
||||
let prompt_message_id = message.id().0 as i64;
|
||||
let ttl_secs = CONFIG.edit_message_ttl.as_secs() as i64;
|
||||
let mut chat_data = CHAT_STORE.get(chat_id).await;
|
||||
let chat_data = CHAT_STORE.get(chat_id).await;
|
||||
let edit = chat_data.edit_message.get(&prompt_message_id).cloned();
|
||||
let Some(edit) = edit else {
|
||||
log::info!(
|
||||
@@ -783,8 +854,11 @@ pub async fn callback_query_handler(bot: Bot, query: CallbackQuery) -> Result<()
|
||||
};
|
||||
// Lazy expiry: a stale record (past the TTL, not yet swept) is dropped.
|
||||
if edit.created_at + ttl_secs <= unix_now() {
|
||||
chat_data.edit_message.remove(&prompt_message_id);
|
||||
CHAT_STORE.set(chat_id, &chat_data).await;
|
||||
CHAT_STORE
|
||||
.update(chat_id, |data| {
|
||||
data.edit_message.remove(&prompt_message_id);
|
||||
})
|
||||
.await;
|
||||
bot.answer_callback_query(callback_query_id)
|
||||
.text("Expired")
|
||||
.await?;
|
||||
@@ -820,8 +894,11 @@ pub async fn callback_query_handler(bot: Bot, query: CallbackQuery) -> Result<()
|
||||
let _ = bot
|
||||
.delete_message(ChatId(chat_id), MessageId(prompt_message_id as i32))
|
||||
.await;
|
||||
chat_data.edit_message.remove(&prompt_message_id);
|
||||
CHAT_STORE.set(chat_id, &chat_data).await;
|
||||
CHAT_STORE
|
||||
.update(chat_id, |data| {
|
||||
data.edit_message.remove(&prompt_message_id);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
Err(send::SendError::Retryable {
|
||||
delay_seconds,
|
||||
@@ -860,10 +937,13 @@ pub async fn callback_query_handler(bot: Bot, query: CallbackQuery) -> Result<()
|
||||
.caption(template_html)
|
||||
.parse_mode(ParseMode::Html)
|
||||
.await;
|
||||
if let Some(entry) = chat_data.edit_message.get_mut(&prompt_message_id) {
|
||||
entry.template = name.to_string();
|
||||
}
|
||||
CHAT_STORE.set(chat_id, &chat_data).await;
|
||||
CHAT_STORE
|
||||
.update(chat_id, |data| {
|
||||
if let Some(entry) = data.edit_message.get_mut(&prompt_message_id) {
|
||||
entry.template = name.to_string();
|
||||
}
|
||||
})
|
||||
.await;
|
||||
log::info!("template '{name}' applied to prompt {prompt_message_id}");
|
||||
}
|
||||
bot.answer_callback_query(callback_query_id).await?;
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
//! [`Config::link_cache_ttl`]; a stale entry is dropped lazily on read and
|
||||
//! by the periodic prune in `main`.
|
||||
|
||||
use crate::db::now_f64;
|
||||
use rusqlite::{Connection, params};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::time::Duration;
|
||||
@@ -163,13 +164,6 @@ impl LinkCache {
|
||||
}
|
||||
}
|
||||
|
||||
fn now_f64() -> f64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_secs_f64())
|
||||
.unwrap_or(0.0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -43,6 +43,14 @@ async fn main() {
|
||||
log::info!("Starting bot");
|
||||
|
||||
let bot = Bot::from_env();
|
||||
// Force the queue workers' shared Bot to initialize now so a missing
|
||||
// token fails at startup, not on the first queued task.
|
||||
let _ = &*send::BOT;
|
||||
|
||||
// Register the command list with Telegram (client `/` menu).
|
||||
if let Err(e) = handlers::register_commands(&bot).await {
|
||||
log::warn!("failed to register commands: {e}");
|
||||
}
|
||||
|
||||
log::info!(
|
||||
"config: {} admin(s), edit-message TTL {}s",
|
||||
@@ -57,6 +65,10 @@ async fn main() {
|
||||
.await;
|
||||
log::info!("task queue worker started");
|
||||
|
||||
// URL job workers: bounded channel + fixed pool for per-URL work.
|
||||
handlers::start_url_workers().await;
|
||||
log::info!("url workers started");
|
||||
|
||||
// Pixiv login validation (user request): a failed login notifies the
|
||||
// admin and disables pixiv for this process.
|
||||
if site::pixiv::enabled() {
|
||||
@@ -169,6 +181,7 @@ async fn main() {
|
||||
// Graceful stop (Ctrl+C / SIGTERM): stop the sweep, notify the admin, drain the queue.
|
||||
log::info!("Stopping bot");
|
||||
let _ = stop_tx.send(true);
|
||||
handlers::stop_url_workers();
|
||||
if let Some(admin) = CONFIG.admin_ids.first() {
|
||||
let _ = bot.send_message(ChatId(*admin), "Shutting down...").await;
|
||||
}
|
||||
|
||||
@@ -26,8 +26,9 @@ pub const PHOTO_TARGET_DIMENSION_SUM: u32 = 9900;
|
||||
/// to a smaller media URL instead.
|
||||
pub const MAX_UPLOAD_BYTES: u64 = 10 * 1024 * 1024;
|
||||
/// Decode budget (bytes): a larger intermediate buffer is not worth the peak
|
||||
/// memory; the photo degrades to the smaller URL instead.
|
||||
const MAX_DECODE_BYTES: u64 = 512 * 1024 * 1024;
|
||||
/// memory; the photo degrades to the smaller URL instead. Also the cap for
|
||||
/// downloading photos in the send fallback (they must be downloaded whole).
|
||||
pub(crate) const MAX_DECODE_BYTES: u64 = 512 * 1024 * 1024;
|
||||
/// JPEG output quality (1-100).
|
||||
const JPEG_QUALITY: u8 = 90;
|
||||
|
||||
|
||||
+139
-36
@@ -5,13 +5,14 @@
|
||||
//! flow. The Python dict-mutation hack (attempts inside the payload) is
|
||||
//! replaced by dedicated columns.
|
||||
|
||||
use crate::db::now_f64;
|
||||
use parking_lot::Mutex;
|
||||
use rusqlite::{Connection, TransactionBehavior, params};
|
||||
use serde_json::Value;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
use std::time::Duration;
|
||||
use tokio::sync::Notify;
|
||||
use tokio::task::JoinHandle;
|
||||
|
||||
@@ -53,6 +54,7 @@ struct LeasedRow {
|
||||
}
|
||||
|
||||
/// Owned worker state so the spawned loop does not borrow the queue handle.
|
||||
#[derive(Clone)]
|
||||
struct QueueWorker {
|
||||
db_path: String,
|
||||
notify: Arc<Notify>,
|
||||
@@ -61,18 +63,31 @@ struct QueueWorker {
|
||||
dead_letter: Arc<DeadLetter>,
|
||||
}
|
||||
|
||||
fn now_f64() -> f64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.as_secs_f64())
|
||||
.unwrap_or(0.0)
|
||||
/// Resets rows left `in_progress` with an expired lock TTL back to `pending`
|
||||
/// so they can be leased again (crash/panic recovery).
|
||||
fn recover_update(conn: &rusqlite::Connection) -> rusqlite::Result<()> {
|
||||
conn.execute(
|
||||
"UPDATE tasks SET status='pending', locked_until=0 WHERE status='in_progress' AND locked_until < ?1",
|
||||
params![now_f64()],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Base delay × 2^attempts (attempts = retries already done), capped at 300s.
|
||||
/// Applied at the queue layer so the attempt count actually reaches the
|
||||
/// backoff computation; Telegram `RetryAfter` delays get the same treatment
|
||||
/// (conservatively larger wait, no API change needed).
|
||||
fn scaled_retry_delay(base: f64, attempts: i32) -> f64 {
|
||||
(base * 2f64.powi(attempts)).min(300.0)
|
||||
}
|
||||
|
||||
fn ensure_schema(conn: &rusqlite::Connection) -> rusqlite::Result<()> {
|
||||
conn.execute_batch(
|
||||
"CREATE TABLE IF NOT EXISTS tasks (id TEXT PRIMARY KEY, payload TEXT NOT NULL, \
|
||||
"PRAGMA journal_mode=WAL; \
|
||||
CREATE TABLE IF NOT EXISTS tasks (id TEXT PRIMARY KEY, payload TEXT NOT NULL, \
|
||||
run_after REAL NOT NULL, attempts INTEGER NOT NULL, status TEXT NOT NULL, \
|
||||
locked_until REAL NOT NULL, created_at REAL NOT NULL);",
|
||||
locked_until REAL NOT NULL, created_at REAL NOT NULL); \
|
||||
CREATE INDEX IF NOT EXISTS idx_tasks_pending ON tasks(status, run_after);",
|
||||
)
|
||||
}
|
||||
|
||||
@@ -114,7 +129,7 @@ impl PersistentTaskQueue {
|
||||
let dead_letter: Arc<DeadLetter> =
|
||||
Arc::new(move |payload, message| Box::pin(dead_letter(payload, message)));
|
||||
self.recover_stale().await;
|
||||
let mut handles = Vec::with_capacity(QUEUE_WORKERS);
|
||||
let mut handles = Vec::with_capacity(QUEUE_WORKERS + 1);
|
||||
for _ in 0..QUEUE_WORKERS {
|
||||
let worker = QueueWorker {
|
||||
db_path: self.db_path.clone(),
|
||||
@@ -123,8 +138,35 @@ impl PersistentTaskQueue {
|
||||
handler: Arc::clone(&handler),
|
||||
dead_letter: Arc::clone(&dead_letter),
|
||||
};
|
||||
handles.push(tokio::spawn(worker.run_loop()));
|
||||
handles.push(tokio::spawn(worker.run_loop_supervised()));
|
||||
}
|
||||
// Periodic lease-expiry sweep: recovers rows a crashed/panicked
|
||||
// worker left `in_progress` (the lock TTL bounds the wait). Woken by
|
||||
// the same notify as the workers, so enqueue and stop interrupt the
|
||||
// sleep; the first interval tick fires immediately (harmless extra
|
||||
// recovery at startup).
|
||||
let sweep_db_path = self.db_path.clone();
|
||||
let sweep_notify = Arc::clone(&self.notify);
|
||||
let sweep_stop = Arc::clone(&self.stop);
|
||||
handles.push(tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(Duration::from_secs(30));
|
||||
loop {
|
||||
let notified = sweep_notify.notified();
|
||||
tokio::pin!(notified);
|
||||
tokio::select! {
|
||||
_ = &mut notified => {}
|
||||
_ = interval.tick() => {}
|
||||
}
|
||||
if sweep_stop.load(Ordering::Relaxed) {
|
||||
break;
|
||||
}
|
||||
let result =
|
||||
crate::db::with_conn(&sweep_db_path, move |conn| recover_update(conn)).await;
|
||||
if let Err(e) = result {
|
||||
log::error!("queue sweep failed: {e}");
|
||||
}
|
||||
}
|
||||
}));
|
||||
*self.worker.lock() = handles;
|
||||
}
|
||||
|
||||
@@ -157,21 +199,20 @@ impl PersistentTaskQueue {
|
||||
Ok(())
|
||||
})
|
||||
.await?;
|
||||
// Wake every sleeping worker: with several workers the one that finds
|
||||
// nothing due must not starve the newly inserted row.
|
||||
self.notify.notify_waiters();
|
||||
// `notify_one` stores a permit when no worker is registered, so a
|
||||
// notification fired between a worker's DB reads and its `notified()`
|
||||
// registration is not lost (notify_waiters would drop it). The
|
||||
// awakened worker re-leases and finds the new row.
|
||||
self.notify.notify_one();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn recover_stale(&self) {
|
||||
let result = crate::db::with_conn(&self.db_path, move |conn| {
|
||||
conn.execute(
|
||||
"UPDATE tasks SET status='pending', locked_until=0 WHERE status='in_progress' AND locked_until < ?1",
|
||||
params![now_f64()],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await;
|
||||
self.recover_sweep().await;
|
||||
}
|
||||
|
||||
async fn recover_sweep(&self) {
|
||||
let result = crate::db::with_conn(&self.db_path, move |conn| recover_update(conn)).await;
|
||||
if let Err(e) = result {
|
||||
log::error!("queue recovery failed: {e}");
|
||||
}
|
||||
@@ -179,11 +220,24 @@ impl PersistentTaskQueue {
|
||||
}
|
||||
|
||||
impl QueueWorker {
|
||||
/// Supervised worker: the inner loop runs in its own task so a panic
|
||||
/// (e.g. inside a handler or a DB closure) kills only that task; the
|
||||
/// supervisor respawns it until stop is set. The row a dead worker had
|
||||
/// leased is recovered by the periodic sweep once its lock TTL expires.
|
||||
async fn run_loop_supervised(self) {
|
||||
while !self.stop.load(Ordering::Relaxed) {
|
||||
let worker = self.clone();
|
||||
if let Err(e) = tokio::spawn(async move { worker.run_loop().await }).await {
|
||||
log::error!("queue worker panicked, restarting: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_loop(self) {
|
||||
while !self.stop.load(Ordering::Relaxed) {
|
||||
match self.lease_next().await {
|
||||
Some(row) => self.process(row).await,
|
||||
None => {
|
||||
Ok(Some(row)) => self.process(row).await,
|
||||
Ok(None) => {
|
||||
let wait_until = self.earliest_run_after().await;
|
||||
let notified = self.notify.notified();
|
||||
tokio::pin!(notified);
|
||||
@@ -200,13 +254,20 @@ impl QueueWorker {
|
||||
}
|
||||
}
|
||||
}
|
||||
// A lease failure while rows are due would otherwise loop
|
||||
// with sleep(0) and hammer SQLite; back off briefly.
|
||||
Err(e) => {
|
||||
log::error!("queue lease failed: {e}");
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Leases the oldest due row (sets it `in_progress` with a lock TTL).
|
||||
async fn lease_next(&self) -> Option<LeasedRow> {
|
||||
let result = crate::db::with_conn(&self.db_path, |conn| {
|
||||
/// Errors are surfaced so the caller can back off instead of spinning.
|
||||
async fn lease_next(&self) -> Result<Option<LeasedRow>, rusqlite::Error> {
|
||||
crate::db::with_conn(&self.db_path, |conn| {
|
||||
// BEGIN IMMEDIATE: with several workers, a deferred transaction
|
||||
// that read before another worker's lease commit would fail with
|
||||
// SQLITE_BUSY_SNAPSHOT. Taking the write lock up front serializes
|
||||
@@ -244,14 +305,7 @@ impl QueueWorker {
|
||||
attempts,
|
||||
}))
|
||||
})
|
||||
.await;
|
||||
match result {
|
||||
Ok(row) => row,
|
||||
Err(e) => {
|
||||
log::error!("queue lease failed: {e}");
|
||||
None
|
||||
}
|
||||
}
|
||||
.await
|
||||
}
|
||||
|
||||
async fn earliest_run_after(&self) -> Option<f64> {
|
||||
@@ -300,12 +354,13 @@ impl QueueWorker {
|
||||
self.delete_row(&row.id).await;
|
||||
(self.dead_letter)(payload, message).await;
|
||||
} else {
|
||||
let delay = scaled_retry_delay(delay_seconds, row.attempts);
|
||||
log::info!(
|
||||
"task {} rescheduled in {delay_seconds:.1}s (attempt {})",
|
||||
"task {} rescheduled in {delay:.1}s (attempt {})",
|
||||
row.id,
|
||||
row.attempts + 1
|
||||
);
|
||||
self.reschedule(&row.id, payload, delay_seconds, row.attempts + 1)
|
||||
self.reschedule(&row.id, payload, delay, row.attempts + 1)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
@@ -343,7 +398,8 @@ impl QueueWorker {
|
||||
if let Err(e) = result {
|
||||
log::error!("queue reschedule failed: {e}");
|
||||
}
|
||||
self.notify.notify_waiters();
|
||||
// Same permit semantics as enqueue: never lose the wakeup.
|
||||
self.notify.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -352,6 +408,16 @@ mod tests {
|
||||
use super::*;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering};
|
||||
|
||||
#[test]
|
||||
fn scaled_retry_delay_scales_and_caps() {
|
||||
assert_eq!(scaled_retry_delay(1.0, 0), 1.0);
|
||||
assert_eq!(scaled_retry_delay(1.0, 1), 2.0);
|
||||
assert_eq!(scaled_retry_delay(1.0, 2), 4.0);
|
||||
assert_eq!(scaled_retry_delay(1.5, 1), 3.0);
|
||||
assert_eq!(scaled_retry_delay(1.0, 10), 300.0, "capped at 300s");
|
||||
assert_eq!(scaled_retry_delay(300.0, 0), 300.0);
|
||||
}
|
||||
|
||||
async fn new_queue() -> (PersistentTaskQueue, tempfile::TempDir) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("queue.db");
|
||||
@@ -490,4 +556,41 @@ mod tests {
|
||||
assert_eq!(calls.load(AtomicOrdering::SeqCst), 1);
|
||||
queue.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn runtime_sweep_recovers_expired_lease() {
|
||||
let (queue, _dir) = new_queue().await;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let c = calls.clone();
|
||||
queue
|
||||
.start(
|
||||
move |payload| {
|
||||
assert_eq!(payload["s"], 1);
|
||||
c.fetch_add(1, AtomicOrdering::SeqCst);
|
||||
async { Ok(()) }
|
||||
},
|
||||
|_payload, _message| async {},
|
||||
)
|
||||
.await;
|
||||
// Insert a stale leased row AFTER startup: without a runtime sweep it
|
||||
// would stay `in_progress` forever (only start() used to recover).
|
||||
{
|
||||
let conn = Connection::open(&queue.db_path).unwrap();
|
||||
ensure_schema(&conn).unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO tasks (id, payload, run_after, attempts, status, locked_until, created_at) \
|
||||
VALUES ('task_stale_runtime', '{\"s\":1}', 0, 0, 'in_progress', ?1, 0)",
|
||||
params![now_f64() - 1000.0],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
queue.recover_sweep().await;
|
||||
tokio::time::sleep(Duration::from_millis(300)).await;
|
||||
assert_eq!(
|
||||
calls.load(AtomicOrdering::SeqCst),
|
||||
1,
|
||||
"expired lease must be recovered and processed exactly once"
|
||||
);
|
||||
queue.stop().await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ use crate::state::{EditMessage, unix_now};
|
||||
use rand::Rng;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::LazyLock;
|
||||
use teloxide::prelude::*;
|
||||
use teloxide::types::{
|
||||
ChatId, InlineKeyboardButton, InlineKeyboardMarkup, InputFile, InputMedia, InputMediaAnimation,
|
||||
@@ -20,6 +21,11 @@ use teloxide::{ApiError, RequestError};
|
||||
use tempfile::NamedTempFile;
|
||||
use x_media::site::FetchError;
|
||||
|
||||
/// One process-wide Bot for queue workers. Building a fresh Bot (and its HTTP
|
||||
/// client) per queue task was pure waste; forced at startup in main so a
|
||||
/// missing token fails fast instead of on the first task.
|
||||
pub static BOT: LazyLock<Bot> = LazyLock::new(Bot::from_env);
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone, Debug)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
pub enum MediaItemPayload {
|
||||
@@ -61,6 +67,15 @@ impl MediaItemPayload {
|
||||
MediaItemPayload::Animation { .. } => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The cover-frame URL for videos (used by the upload fallback, which
|
||||
/// otherwise drops the thumbnail the URL-send path applies).
|
||||
fn thumbnail_url(&self) -> Option<&str> {
|
||||
match self {
|
||||
MediaItemPayload::Video { thumbnail, .. } => thumbnail.as_deref(),
|
||||
MediaItemPayload::Photo { .. } | MediaItemPayload::Animation { .. } => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone, Debug)]
|
||||
@@ -329,6 +344,11 @@ fn item_url(item: &MediaItemPayload) -> &str {
|
||||
fn input_file_for(media: &str) -> Result<InputFile, String> {
|
||||
if media.starts_with("http://") || media.starts_with("https://") {
|
||||
Ok(InputFile::url(parse_media_url(media)?))
|
||||
} else if !std::path::Path::new(media).exists() {
|
||||
// A retried task may reference a temp file the original send's
|
||||
// TempDir already cleaned up; fail fast and permanent instead of
|
||||
// burning retries on a file that can never come back.
|
||||
Err(format!("local media file missing: {media}"))
|
||||
} else {
|
||||
Ok(InputFile::file(media))
|
||||
}
|
||||
@@ -472,24 +492,30 @@ async fn download_to_temp(item: &MediaItemPayload) -> Result<NamedTempFile, Fall
|
||||
| MediaItemPayload::Video { media, .. }
|
||||
| MediaItemPayload::Animation { media, .. } => media,
|
||||
};
|
||||
let bytes = match x_media::site::download_media(media_url).await {
|
||||
// Photos are downloaded even over the upload cap so `prepare_photo` can
|
||||
// downscale / transcode them (cap = decode budget); videos/animations
|
||||
// abort as soon as the upload cap is crossed mid-stream.
|
||||
let limit = if matches!(item, MediaItemPayload::Photo { .. }) {
|
||||
photo::MAX_DECODE_BYTES
|
||||
} else {
|
||||
MAX_UPLOAD_BYTES + 1
|
||||
};
|
||||
let bytes = match x_media::site::download_media_limited(media_url, limit).await {
|
||||
Ok(bytes) => bytes,
|
||||
Err(FetchError::Http(_)) => {
|
||||
return Err(FallbackError::Retryable {
|
||||
delay_seconds: retry_delay_seconds(0),
|
||||
});
|
||||
}
|
||||
Err(FetchError::TooLarge) => {
|
||||
return Err(FallbackError::MediaTooLarge);
|
||||
}
|
||||
Err(e) => {
|
||||
return Err(FallbackError::Permanent {
|
||||
message: format!("download failed: {e}"),
|
||||
});
|
||||
}
|
||||
};
|
||||
// Photos are downloaded even over the cap so `prepare_photo` can
|
||||
// downscale / transcode them; only videos/animations short-circuit.
|
||||
if !matches!(item, MediaItemPayload::Photo { .. }) && bytes.len() as u64 > MAX_UPLOAD_BYTES {
|
||||
return Err(FallbackError::MediaTooLarge);
|
||||
}
|
||||
let ext = sniff_ext(&bytes);
|
||||
let mut file = tempfile::Builder::new()
|
||||
.suffix(&format!(".{ext}"))
|
||||
@@ -511,8 +537,9 @@ fn media_from_file(
|
||||
item: &MediaItemPayload,
|
||||
path: std::path::PathBuf,
|
||||
caption: Option<&str>,
|
||||
) -> InputMedia {
|
||||
match item {
|
||||
thumbnail: Option<&str>,
|
||||
) -> Result<InputMedia, String> {
|
||||
let mut media = match item {
|
||||
MediaItemPayload::Photo { has_spoiler, .. } => {
|
||||
photo_media(InputFile::file(path), caption, *has_spoiler)
|
||||
}
|
||||
@@ -522,7 +549,11 @@ fn media_from_file(
|
||||
MediaItemPayload::Animation { has_spoiler, .. } => {
|
||||
animation_media(InputFile::file(path), caption, *has_spoiler)
|
||||
}
|
||||
};
|
||||
if let (Some(thumb), InputMedia::Video(v)) = (thumbnail, &mut media) {
|
||||
*v = v.clone().thumbnail(input_file_for(thumb)?);
|
||||
}
|
||||
Ok(media)
|
||||
}
|
||||
|
||||
/// Builds the media group item from a (smaller) URL.
|
||||
@@ -530,8 +561,9 @@ fn media_from_url(
|
||||
item: &MediaItemPayload,
|
||||
url: &str,
|
||||
caption: Option<&str>,
|
||||
thumbnail: Option<&str>,
|
||||
) -> Result<InputMedia, String> {
|
||||
Ok(match item {
|
||||
let mut media = match item {
|
||||
MediaItemPayload::Photo { has_spoiler, .. } => {
|
||||
photo_media(input_file_for(url)?, caption, *has_spoiler)
|
||||
}
|
||||
@@ -541,7 +573,11 @@ fn media_from_url(
|
||||
MediaItemPayload::Animation { has_spoiler, .. } => {
|
||||
animation_media(input_file_for(url)?, caption, *has_spoiler)
|
||||
}
|
||||
})
|
||||
};
|
||||
if let (Some(thumb), InputMedia::Video(v)) = (thumbnail, &mut media) {
|
||||
*v = v.clone().thumbnail(input_file_for(thumb)?);
|
||||
}
|
||||
Ok(media)
|
||||
}
|
||||
|
||||
/// Download-and-reupload fallback for one media batch. Files over the upload
|
||||
@@ -569,7 +605,7 @@ async fn send_batch_via_upload(
|
||||
let too_large = too_large && !matches!(item, MediaItemPayload::Photo { .. });
|
||||
let media = if too_large {
|
||||
match item.fallback_url() {
|
||||
Some(url) => match media_from_url(item, url, item_caption) {
|
||||
Some(url) => match media_from_url(item, url, item_caption, item.thumbnail_url()) {
|
||||
Ok(media) => media,
|
||||
Err(message) => {
|
||||
return Err(FallbackError::Permanent { message });
|
||||
@@ -601,10 +637,16 @@ async fn send_batch_via_upload(
|
||||
PhotoPrep::Upload(upload) => {
|
||||
let path = upload.path().to_path_buf();
|
||||
files.push(upload);
|
||||
media_from_file(item, path, item_caption)
|
||||
media_from_file(item, path, item_caption, item.thumbnail_url())
|
||||
.map_err(|message| FallbackError::Permanent { message })?
|
||||
}
|
||||
PhotoPrep::UseFallback => match item.fallback_url() {
|
||||
Some(url) => match media_from_url(item, url, item_caption) {
|
||||
Some(url) => match media_from_url(
|
||||
item,
|
||||
url,
|
||||
item_caption,
|
||||
item.thumbnail_url(),
|
||||
) {
|
||||
Ok(media) => media,
|
||||
Err(message) => {
|
||||
return Err(FallbackError::Permanent { message });
|
||||
@@ -622,16 +664,19 @@ async fn send_batch_via_upload(
|
||||
} else {
|
||||
let path = file.path().to_path_buf();
|
||||
files.push(file);
|
||||
media_from_file(item, path, item_caption)
|
||||
media_from_file(item, path, item_caption, item.thumbnail_url())
|
||||
.map_err(|message| FallbackError::Permanent { message })?
|
||||
}
|
||||
}
|
||||
Err(FallbackError::MediaTooLarge) => match item.fallback_url() {
|
||||
Some(url) => match media_from_url(item, url, item_caption) {
|
||||
Ok(media) => media,
|
||||
Err(message) => {
|
||||
return Err(FallbackError::Permanent { message });
|
||||
Some(url) => {
|
||||
match media_from_url(item, url, item_caption, item.thumbnail_url()) {
|
||||
Ok(media) => media,
|
||||
Err(message) => {
|
||||
return Err(FallbackError::Permanent { message });
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
None => {
|
||||
return Err(FallbackError::Permanent {
|
||||
message: "media too large".into(),
|
||||
@@ -1037,8 +1082,7 @@ pub async fn post_send_actions(bot: &Bot, task: &Task, message_ids: Vec<i64>) {
|
||||
};
|
||||
|
||||
if edit_before_forward {
|
||||
let mut chat_data = CHAT_STORE.get(chat_id).await;
|
||||
let keyboard = build_edit_markup(&chat_data.template);
|
||||
let keyboard = build_edit_markup(&CHAT_STORE.get(chat_id).await.template);
|
||||
match bot
|
||||
.send_message(ChatId(chat_id), "Reply to edit message.")
|
||||
.reply_markup(keyboard)
|
||||
@@ -1053,17 +1097,22 @@ pub async fn post_send_actions(bot: &Bot, task: &Task, message_ids: Vec<i64>) {
|
||||
prompt.id.0,
|
||||
message_ids.len()
|
||||
);
|
||||
chat_data.edit_message.insert(
|
||||
prompt.id.0 as i64,
|
||||
EditMessage {
|
||||
url: source_url,
|
||||
chat_id,
|
||||
forward_message_ids: message_ids,
|
||||
template: String::new(),
|
||||
created_at: unix_now(),
|
||||
},
|
||||
);
|
||||
CHAT_STORE.set(chat_id, &chat_data).await;
|
||||
let prompt_id = prompt.id.0 as i64;
|
||||
let source_url = source_url.clone();
|
||||
CHAT_STORE
|
||||
.update(chat_id, move |data| {
|
||||
data.edit_message.insert(
|
||||
prompt_id,
|
||||
EditMessage {
|
||||
url: source_url,
|
||||
chat_id,
|
||||
forward_message_ids: message_ids,
|
||||
template: String::new(),
|
||||
created_at: unix_now(),
|
||||
},
|
||||
);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
Err(e) => log::error!("failed to send edit prompt: {e}"),
|
||||
}
|
||||
@@ -1122,7 +1171,19 @@ pub async fn handle_task(payload: serde_json::Value) -> Result<(), QueueError> {
|
||||
});
|
||||
}
|
||||
};
|
||||
let bot = Bot::from_env();
|
||||
let bot = BOT.clone();
|
||||
// A resumed multi-batch send already ran post_send_actions (edit prompt /
|
||||
// forward) when it first started; running them again on the resume would
|
||||
// open a duplicate edit prompt and double-forward. SendAnimation is
|
||||
// atomic (always a fresh run), so only SendMediaSequence can resume.
|
||||
let resumed = matches!(
|
||||
&task,
|
||||
Task::SendMediaSequence {
|
||||
batch_index,
|
||||
sent_message_ids,
|
||||
..
|
||||
} if *batch_index > 0 || !sent_message_ids.is_empty()
|
||||
);
|
||||
match task {
|
||||
Task::SendMediaSequence { .. } | Task::SendAnimation { .. } => {
|
||||
let message_ids = match send_media_or_animation(&bot, &task).await {
|
||||
@@ -1144,7 +1205,9 @@ pub async fn handle_task(payload: serde_json::Value) -> Result<(), QueueError> {
|
||||
});
|
||||
}
|
||||
};
|
||||
post_send_actions(&bot, &task, message_ids).await;
|
||||
if !resumed {
|
||||
post_send_actions(&bot, &task, message_ids).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Task::ForwardMessages { .. } => match forward_messages(&bot, &task).await {
|
||||
@@ -1177,7 +1240,7 @@ pub async fn dead_letter_notify(payload: serde_json::Value, message: String) {
|
||||
let notify_chat_id = payload.get("notify_chat_id").and_then(|v| v.as_i64());
|
||||
let notify_message_id = payload.get("notify_message_id").and_then(|v| v.as_i64());
|
||||
if notify_chat_id.is_some() {
|
||||
let bot = Bot::from_env();
|
||||
let bot = BOT.clone();
|
||||
notify_failure(
|
||||
&bot,
|
||||
notify_chat_id,
|
||||
|
||||
@@ -6,6 +6,7 @@ use rusqlite::params;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
#[derive(Serialize, Deserialize, Default, Clone, Debug)]
|
||||
@@ -34,6 +35,9 @@ pub struct EditMessage {
|
||||
pub struct ChatStore {
|
||||
/// In-memory cache; the DB is the source of truth on first access.
|
||||
cache: Mutex<HashMap<i64, ChatData>>,
|
||||
/// Per-chat async locks serializing get→mutate→set so concurrent handler
|
||||
/// tasks (batch-forwards, callbacks) cannot clobber each other's writes.
|
||||
locks: Mutex<HashMap<i64, Arc<tokio::sync::Mutex<()>>>>,
|
||||
db_path: String,
|
||||
}
|
||||
|
||||
@@ -62,6 +66,7 @@ impl ChatStore {
|
||||
drop(conn);
|
||||
Ok(ChatStore {
|
||||
cache: Mutex::new(HashMap::new()),
|
||||
locks: Mutex::new(HashMap::new()),
|
||||
db_path: path.to_string(),
|
||||
})
|
||||
}
|
||||
@@ -111,6 +116,26 @@ impl ChatStore {
|
||||
}
|
||||
}
|
||||
|
||||
/// Serializes a get→mutate→set cycle per chat: concurrent handler tasks
|
||||
/// (the batch-forward design spawns several per chat) each snapshot the
|
||||
/// same `ChatData` and last-writer-wins would silently drop mutations,
|
||||
/// e.g. a second `edit_message` record. The per-chat lock makes the
|
||||
/// cycle atomic. Returns the closure's result.
|
||||
pub async fn update<R>(&self, chat_id: i64, f: impl FnOnce(&mut ChatData) -> R) -> R {
|
||||
let lock = {
|
||||
let mut locks = self.locks.lock();
|
||||
locks
|
||||
.entry(chat_id)
|
||||
.or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
|
||||
.clone()
|
||||
};
|
||||
let _guard = lock.lock().await;
|
||||
let mut data = self.get(chat_id).await;
|
||||
let r = f(&mut data);
|
||||
self.set(chat_id, &data).await;
|
||||
r
|
||||
}
|
||||
|
||||
/// Removes edit-before-forward records whose `created_at + ttl` is in the
|
||||
/// past. Returns the removed `(chat_id, prompt_message_id)` pairs so the
|
||||
/// caller can clear the prompt's buttons.
|
||||
@@ -118,6 +143,10 @@ impl ChatStore {
|
||||
let now = unix_now();
|
||||
let ttl_secs = ttl.as_secs() as i64;
|
||||
let mut removed = Vec::new();
|
||||
// Chats with no live edit records: evicted from the cache (and their
|
||||
// per-chat lock) so the cache stays bounded to active prompts. The DB
|
||||
// keeps the row; the next get() reloads it.
|
||||
let mut evicted_chats = Vec::new();
|
||||
let changed: Vec<(i64, ChatData)> = {
|
||||
let mut cache = self.cache.lock();
|
||||
let mut out = Vec::new();
|
||||
@@ -134,15 +163,31 @@ impl ChatStore {
|
||||
}
|
||||
}
|
||||
if kept.len() != data.edit_message.len() {
|
||||
// Persist the pruned row (removes expired records from
|
||||
// the DB too, not just the cache).
|
||||
data.edit_message = kept;
|
||||
out.push((*chat_id, data.clone()));
|
||||
}
|
||||
if data.edit_message.is_empty() {
|
||||
evicted_chats.push(*chat_id);
|
||||
}
|
||||
}
|
||||
// Lock order: update() takes the per-chat lock before the cache
|
||||
// lock, so prune must not hold the cache lock while taking locks.
|
||||
drop(cache);
|
||||
out
|
||||
};
|
||||
for (chat_id, data) in changed {
|
||||
self.set(chat_id, &data).await;
|
||||
}
|
||||
if !evicted_chats.is_empty() {
|
||||
let mut cache = self.cache.lock();
|
||||
let mut locks = self.locks.lock();
|
||||
for chat_id in &evicted_chats {
|
||||
cache.remove(chat_id);
|
||||
locks.remove(chat_id);
|
||||
}
|
||||
}
|
||||
if !removed.is_empty() {
|
||||
log::info!(
|
||||
"pruned {} expired edit-before-forward record(s)",
|
||||
@@ -152,3 +197,45 @@ impl ChatStore {
|
||||
removed
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_updates_do_not_lose_edit_records() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let store = std::sync::Arc::new(
|
||||
ChatStore::open(dir.path().join("s.db").to_str().unwrap()).unwrap(),
|
||||
);
|
||||
let mut handles = Vec::new();
|
||||
for i in 0..4 {
|
||||
let store = Arc::clone(&store);
|
||||
handles.push(tokio::spawn(async move {
|
||||
store
|
||||
.update(1001, |data| {
|
||||
data.edit_message.insert(
|
||||
i,
|
||||
EditMessage {
|
||||
url: format!("https://x.com/u/status/{i}"),
|
||||
chat_id: 1001,
|
||||
forward_message_ids: vec![i],
|
||||
template: String::new(),
|
||||
created_at: 0,
|
||||
},
|
||||
);
|
||||
})
|
||||
.await;
|
||||
}));
|
||||
}
|
||||
for h in handles {
|
||||
h.await.unwrap();
|
||||
}
|
||||
let data = store.get(1001).await;
|
||||
assert_eq!(
|
||||
data.edit_message.len(),
|
||||
4,
|
||||
"concurrent get→mutate→set must not drop records"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,6 +55,14 @@ services:
|
||||
depends_on:
|
||||
- nginx-proxy
|
||||
container_name: tgxmb
|
||||
# Webhook mode only: the bot listens on WEBHOOK_PORT; nginx-proxy shows
|
||||
# 502s while this is down, so surface it to the orchestrator.
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "bash -c 'exec 3<>/dev/tcp/127.0.0.1/8443'"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
|
||||
volumes:
|
||||
certs:
|
||||
|
||||
@@ -16,7 +16,6 @@ then
|
||||
else
|
||||
usermod -u ${USER_ID} -o user > /dev/null 2>&1 || true
|
||||
fi
|
||||
usermod -a -G root user > /dev/null 2>&1 || true
|
||||
# Bind-mounted volumes may not support chown; a failure here must not kill
|
||||
# the container either.
|
||||
chown -R `id -u user`:`id -u user` /app > /dev/null 2>&1 || true
|
||||
|
||||
Reference in New Issue
Block a user