mirror of
https://github.com/TheFunny/TelegramTwitterMediaBot.git
synced 2026-09-23 23:32:05 +00:00
fix(twitter): decode API HTML entities so captions escape exactly once
Twitter's syndication and GraphQL APIs return tweet text and display names pre-escaped for HTML (> < & '); the caption builder escaped the text again, so sent messages showed literal entities (e.g. >^ω^< came back as >^ω^<). from_syndication_json now decodes the API text before storing it — both the syndication path and the TWITTER_AUTH_TOKEN GraphQL fallback route through it — so the caption escapes exactly once and renders correctly. The /test report is a plain-text message but printed the pre-escaped caption and render fields; it now HTML-decodes them for display so the report shows the rendered text.
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
use super::model;
|
||||
use crate::media::Media;
|
||||
use crate::site::{FetchError, Fetched, Site, SiteFuture};
|
||||
use html_escape::{encode_double_quoted_attribute, encode_text};
|
||||
use html_escape::{decode_html_entities, encode_double_quoted_attribute, encode_text};
|
||||
use regex::Regex;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
@@ -239,9 +239,17 @@ impl Tweet {
|
||||
// strip the appended media short link, mirroring FxEmbed's linkFixer
|
||||
// (no display_text_range arithmetic — see expand_links).
|
||||
let text = expand_links(&json.text, &json.entities.urls);
|
||||
// Twitter APIs (syndication AND GraphQL full_text) return the text
|
||||
// pre-escaped for HTML (`>` `<` `&` `'` …): decode it so
|
||||
// the stored text is raw. The caption's own escaping then produces
|
||||
// the rendered form exactly once — without this, `>^ω^<` would
|
||||
// be double-escaped to `&gt;^ω^&lt;` and the sent message
|
||||
// would show literal `>^ω^<`.
|
||||
let text = decode_html_entities(&text).into_owned();
|
||||
// `name` is the display name, `screen_name` the handle (Python's
|
||||
// vxtwitter mapping: author = display name, author_id = handle).
|
||||
let author = json.user.name;
|
||||
// Display names can carry the same pre-escaped entities.
|
||||
let author = decode_html_entities(&json.user.name).into_owned();
|
||||
let author_id = json.user.screen_name;
|
||||
let mut media = vec![];
|
||||
for item in json.media_details {
|
||||
@@ -409,6 +417,41 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn syndication_text_is_unescaped_before_storing() {
|
||||
// Real API shape: the text arrives pre-escaped for HTML — e.g. the
|
||||
// tweet `>^ω^<` comes back as `>^ω^<` (fxtwitter's raw_text for
|
||||
// 2060196388252827954) and apostrophes as `'`. Storing it raw and
|
||||
// escaping once at caption build avoids the double-escape that would
|
||||
// show literal `>`/`<`/`&` in the sent message.
|
||||
let raw = serde_json::json!({
|
||||
"__typename": "Tweet",
|
||||
"id_str": "1",
|
||||
"text": ">^ω^< & more 'quoted' https://t.co/abc123",
|
||||
"user": { "name": "O'Brien", "screen_name": "h" },
|
||||
"entities": { "urls": [] },
|
||||
"mediaDetails": []
|
||||
});
|
||||
let tweet = Tweet::from_syndication_json(&raw.to_string()).unwrap();
|
||||
// The appended media short link is stripped, then entities decoded.
|
||||
assert_eq!(tweet.text, ">^ω^< & more 'quoted'");
|
||||
assert_eq!(tweet.author, "O'Brien");
|
||||
let fetched: Fetched = tweet.into();
|
||||
assert_eq!(fetched.title, ">^ω^< & more 'quoted'");
|
||||
// The caption escapes the raw text exactly once (encode_text covers
|
||||
// & < >; apostrophes stay literal — they are harmless in text).
|
||||
assert!(
|
||||
fetched.caption.contains(">^ω^< & more 'quoted'"),
|
||||
"caption: {}",
|
||||
fetched.caption
|
||||
);
|
||||
assert!(
|
||||
!fetched.caption.contains("&gt;"),
|
||||
"double-escaped text: {}",
|
||||
fetched.caption
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cache_key_prefixes_tweet_id() {
|
||||
assert_eq!(
|
||||
|
||||
@@ -415,14 +415,20 @@ fn test_parse_report(
|
||||
lines.push(format!("source_url: {source_url}"));
|
||||
lines.push(format!("title: {title}"));
|
||||
if let Some((author, author_url, _title, tags)) = render {
|
||||
lines.push(format!("author: {author}"));
|
||||
// The render fields are pre-escaped for HTML captions; decode them
|
||||
// so the plain-text report shows the text as it will be rendered
|
||||
// (no visible & / < / >).
|
||||
lines.push(format!(
|
||||
"author: {}",
|
||||
html_escape::decode_html_entities(author)
|
||||
));
|
||||
lines.push(format!("author_url: {author_url}"));
|
||||
lines.push(format!("tags: {tags}"));
|
||||
lines.push(format!("tags: {}", html_escape::decode_html_entities(tags)));
|
||||
}
|
||||
lines.push(format!("sensitive: {sensitive}"));
|
||||
lines.push(format!(
|
||||
"caption: {}",
|
||||
x_media::site::truncate_caption(caption)
|
||||
x_media::site::truncate_caption(&html_escape::decode_html_entities(caption))
|
||||
));
|
||||
lines.push(format!("media ({}):", media.len()));
|
||||
for (i, item) in media.iter().enumerate() {
|
||||
@@ -500,6 +506,38 @@ mod tests {
|
||||
assert!(report.contains("media (0):"), "{report}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_report_decodes_html_entities_for_display() {
|
||||
// The report is a plain-text message: pre-escaped caption fields and
|
||||
// the HTML caption must be shown decoded (as rendered), never with
|
||||
// visible & / < / >.
|
||||
let report = test_parse_report(
|
||||
"https://x.com/u/status/1",
|
||||
"twitter",
|
||||
"https://x.com/u/status/1",
|
||||
"A & B <C>",
|
||||
Some((
|
||||
"A & B",
|
||||
"https://x.com/u",
|
||||
"A & B <C>",
|
||||
"#a & #b",
|
||||
)),
|
||||
false,
|
||||
"<a href=\"https://x.com/u\">A & B</a>: C <D> & E",
|
||||
&[],
|
||||
);
|
||||
assert!(report.contains("title: A & B <C>"), "{report}");
|
||||
assert!(report.contains("author: A & B"), "{report}");
|
||||
assert!(report.contains("tags: #a & #b"), "{report}");
|
||||
assert!(
|
||||
report.contains("caption: <a href=\"https://x.com/u\">A & B</a>: C <D> & E"),
|
||||
"{report}"
|
||||
);
|
||||
for entity in ["&", "<", ">"] {
|
||||
assert!(!report.contains(entity), "unexpected {entity} in: {report}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_report_is_capped() {
|
||||
// 200 media lines ≈ 8 KB, comfortably over the cap.
|
||||
|
||||
Reference in New Issue
Block a user