From af901caddb6fb15d1cab60a45329f672484e9144 Mon Sep 17 00:00:00 2001 From: YoursFunny Date: Sun, 16 Aug 2026 16:31:16 +0800 Subject: [PATCH] fix(twitter): decode API HTML entities so captions escape exactly once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Twitter's syndication and GraphQL APIs return tweet text and display names pre-escaped for HTML (> < & '); the caption builder escaped the text again, so sent messages showed literal entities (e.g. >^ω^< came back as >^ω^<). from_syndication_json now decodes the API text before storing it — both the syndication path and the TWITTER_AUTH_TOKEN GraphQL fallback route through it — so the caption escapes exactly once and renders correctly. The /test report is a plain-text message but printed the pre-escaped caption and render fields; it now HTML-decodes them for display so the report shows the rendered text. --- crates/x-media/src/site/twitter/interface.rs | 47 +++++++++++++++++++- crates/xmedia-bot/src/handlers/commands.rs | 44 ++++++++++++++++-- 2 files changed, 86 insertions(+), 5 deletions(-) diff --git a/crates/x-media/src/site/twitter/interface.rs b/crates/x-media/src/site/twitter/interface.rs index e945b5d..31b7d8a 100644 --- a/crates/x-media/src/site/twitter/interface.rs +++ b/crates/x-media/src/site/twitter/interface.rs @@ -1,7 +1,7 @@ use super::model; use crate::media::Media; use crate::site::{FetchError, Fetched, Site, SiteFuture}; -use html_escape::{encode_double_quoted_attribute, encode_text}; +use html_escape::{decode_html_entities, encode_double_quoted_attribute, encode_text}; use regex::Regex; use std::sync::LazyLock; @@ -239,9 +239,17 @@ impl Tweet { // strip the appended media short link, mirroring FxEmbed's linkFixer // (no display_text_range arithmetic — see expand_links). let text = expand_links(&json.text, &json.entities.urls); + // Twitter APIs (syndication AND GraphQL full_text) return the text + // pre-escaped for HTML (`>` `<` `&` `'` …): decode it so + // the stored text is raw. The caption's own escaping then produces + // the rendered form exactly once — without this, `>^ω^<` would + // be double-escaped to `&gt;^ω^&lt;` and the sent message + // would show literal `>^ω^<`. + let text = decode_html_entities(&text).into_owned(); // `name` is the display name, `screen_name` the handle (Python's // vxtwitter mapping: author = display name, author_id = handle). - let author = json.user.name; + // Display names can carry the same pre-escaped entities. + let author = decode_html_entities(&json.user.name).into_owned(); let author_id = json.user.screen_name; let mut media = vec![]; for item in json.media_details { @@ -409,6 +417,41 @@ mod tests { } } + #[test] + fn syndication_text_is_unescaped_before_storing() { + // Real API shape: the text arrives pre-escaped for HTML — e.g. the + // tweet `>^ω^<` comes back as `>^ω^<` (fxtwitter's raw_text for + // 2060196388252827954) and apostrophes as `'`. Storing it raw and + // escaping once at caption build avoids the double-escape that would + // show literal `>`/`<`/`&` in the sent message. + let raw = serde_json::json!({ + "__typename": "Tweet", + "id_str": "1", + "text": ">^ω^< & more 'quoted' https://t.co/abc123", + "user": { "name": "O'Brien", "screen_name": "h" }, + "entities": { "urls": [] }, + "mediaDetails": [] + }); + let tweet = Tweet::from_syndication_json(&raw.to_string()).unwrap(); + // The appended media short link is stripped, then entities decoded. + assert_eq!(tweet.text, ">^ω^< & more 'quoted'"); + assert_eq!(tweet.author, "O'Brien"); + let fetched: Fetched = tweet.into(); + assert_eq!(fetched.title, ">^ω^< & more 'quoted'"); + // The caption escapes the raw text exactly once (encode_text covers + // & < >; apostrophes stay literal — they are harmless in text). + assert!( + fetched.caption.contains(">^ω^< & more 'quoted'"), + "caption: {}", + fetched.caption + ); + assert!( + !fetched.caption.contains("&gt;"), + "double-escaped text: {}", + fetched.caption + ); + } + #[test] fn cache_key_prefixes_tweet_id() { assert_eq!( diff --git a/crates/xmedia-bot/src/handlers/commands.rs b/crates/xmedia-bot/src/handlers/commands.rs index 1d18ecd..9ef39d8 100644 --- a/crates/xmedia-bot/src/handlers/commands.rs +++ b/crates/xmedia-bot/src/handlers/commands.rs @@ -415,14 +415,20 @@ fn test_parse_report( lines.push(format!("source_url: {source_url}")); lines.push(format!("title: {title}")); if let Some((author, author_url, _title, tags)) = render { - lines.push(format!("author: {author}")); + // The render fields are pre-escaped for HTML captions; decode them + // so the plain-text report shows the text as it will be rendered + // (no visible & / < / >). + lines.push(format!( + "author: {}", + html_escape::decode_html_entities(author) + )); lines.push(format!("author_url: {author_url}")); - lines.push(format!("tags: {tags}")); + lines.push(format!("tags: {}", html_escape::decode_html_entities(tags))); } lines.push(format!("sensitive: {sensitive}")); lines.push(format!( "caption: {}", - x_media::site::truncate_caption(caption) + x_media::site::truncate_caption(&html_escape::decode_html_entities(caption)) )); lines.push(format!("media ({}):", media.len())); for (i, item) in media.iter().enumerate() { @@ -500,6 +506,38 @@ mod tests { assert!(report.contains("media (0):"), "{report}"); } + #[test] + fn test_parse_report_decodes_html_entities_for_display() { + // The report is a plain-text message: pre-escaped caption fields and + // the HTML caption must be shown decoded (as rendered), never with + // visible & / < / >. + let report = test_parse_report( + "https://x.com/u/status/1", + "twitter", + "https://x.com/u/status/1", + "A & B ", + Some(( + "A & B", + "https://x.com/u", + "A & B <C>", + "#a & #b", + )), + false, + "A & B: C <D> & E", + &[], + ); + assert!(report.contains("title: A & B "), "{report}"); + assert!(report.contains("author: A & B"), "{report}"); + assert!(report.contains("tags: #a & #b"), "{report}"); + assert!( + report.contains("caption: A & B: C & E"), + "{report}" + ); + for entity in ["&", "<", ">"] { + assert!(!report.contains(entity), "unexpected {entity} in: {report}"); + } + } + #[test] fn test_parse_report_is_capped() { // 200 media lines ≈ 8 KB, comfortably over the cap.