fix(twitter): decode API HTML entities so captions escape exactly once

Twitter's syndication and GraphQL APIs return tweet text and display
names pre-escaped for HTML (> < & '); the caption builder
escaped the text again, so sent messages showed literal entities (e.g.
>^ω^< came back as &gt;^ω^&lt;). from_syndication_json now decodes the
API text before storing it — both the syndication path and the
TWITTER_AUTH_TOKEN GraphQL fallback route through it — so the caption
escapes exactly once and renders correctly.

The /test report is a plain-text message but printed the pre-escaped
caption and render fields; it now HTML-decodes them for display so the
report shows the rendered text.
This commit is contained in:
2026-08-16 16:31:16 +08:00
parent c0af42b1cc
commit af901caddb
2 changed files with 86 additions and 5 deletions
+45 -2
View File
@@ -1,7 +1,7 @@
use super::model;
use crate::media::Media;
use crate::site::{FetchError, Fetched, Site, SiteFuture};
use html_escape::{encode_double_quoted_attribute, encode_text};
use html_escape::{decode_html_entities, encode_double_quoted_attribute, encode_text};
use regex::Regex;
use std::sync::LazyLock;
@@ -239,9 +239,17 @@ impl Tweet {
// strip the appended media short link, mirroring FxEmbed's linkFixer
// (no display_text_range arithmetic — see expand_links).
let text = expand_links(&json.text, &json.entities.urls);
// Twitter APIs (syndication AND GraphQL full_text) return the text
// pre-escaped for HTML (`&gt;` `&lt;` `&amp;` `&#39;` …): decode it so
// the stored text is raw. The caption's own escaping then produces
// the rendered form exactly once — without this, `&gt;^ω^&lt;` would
// be double-escaped to `&amp;gt;^ω^&amp;lt;` and the sent message
// would show literal `&gt;^ω^&lt;`.
let text = decode_html_entities(&text).into_owned();
// `name` is the display name, `screen_name` the handle (Python's
// vxtwitter mapping: author = display name, author_id = handle).
let author = json.user.name;
// Display names can carry the same pre-escaped entities.
let author = decode_html_entities(&json.user.name).into_owned();
let author_id = json.user.screen_name;
let mut media = vec![];
for item in json.media_details {
@@ -409,6 +417,41 @@ mod tests {
}
}
#[test]
fn syndication_text_is_unescaped_before_storing() {
// Real API shape: the text arrives pre-escaped for HTML — e.g. the
// tweet `>^ω^<` comes back as `&gt;^ω^&lt;` (fxtwitter's raw_text for
// 2060196388252827954) and apostrophes as `&#39;`. Storing it raw and
// escaping once at caption build avoids the double-escape that would
// show literal `&gt;`/`&lt;`/`&amp;` in the sent message.
let raw = serde_json::json!({
"__typename": "Tweet",
"id_str": "1",
"text": "&gt;^ω^&lt; &amp; more &#39;quoted&#39; https://t.co/abc123",
"user": { "name": "O&#39;Brien", "screen_name": "h" },
"entities": { "urls": [] },
"mediaDetails": []
});
let tweet = Tweet::from_syndication_json(&raw.to_string()).unwrap();
// The appended media short link is stripped, then entities decoded.
assert_eq!(tweet.text, ">^ω^< & more 'quoted'");
assert_eq!(tweet.author, "O'Brien");
let fetched: Fetched = tweet.into();
assert_eq!(fetched.title, ">^ω^< & more 'quoted'");
// The caption escapes the raw text exactly once (encode_text covers
// & < >; apostrophes stay literal — they are harmless in text).
assert!(
fetched.caption.contains("&gt;^ω^&lt; &amp; more 'quoted'"),
"caption: {}",
fetched.caption
);
assert!(
!fetched.caption.contains("&amp;gt;"),
"double-escaped text: {}",
fetched.caption
);
}
#[test]
fn cache_key_prefixes_tweet_id() {
assert_eq!(
+41 -3
View File
@@ -415,14 +415,20 @@ fn test_parse_report(
lines.push(format!("source_url: {source_url}"));
lines.push(format!("title: {title}"));
if let Some((author, author_url, _title, tags)) = render {
lines.push(format!("author: {author}"));
// The render fields are pre-escaped for HTML captions; decode them
// so the plain-text report shows the text as it will be rendered
// (no visible &amp; / &lt; / &gt;).
lines.push(format!(
"author: {}",
html_escape::decode_html_entities(author)
));
lines.push(format!("author_url: {author_url}"));
lines.push(format!("tags: {tags}"));
lines.push(format!("tags: {}", html_escape::decode_html_entities(tags)));
}
lines.push(format!("sensitive: {sensitive}"));
lines.push(format!(
"caption: {}",
x_media::site::truncate_caption(caption)
x_media::site::truncate_caption(&html_escape::decode_html_entities(caption))
));
lines.push(format!("media ({}):", media.len()));
for (i, item) in media.iter().enumerate() {
@@ -500,6 +506,38 @@ mod tests {
assert!(report.contains("media (0):"), "{report}");
}
#[test]
fn test_parse_report_decodes_html_entities_for_display() {
// The report is a plain-text message: pre-escaped caption fields and
// the HTML caption must be shown decoded (as rendered), never with
// visible &amp; / &lt; / &gt;.
let report = test_parse_report(
"https://x.com/u/status/1",
"twitter",
"https://x.com/u/status/1",
"A & B <C>",
Some((
"A &amp; B",
"https://x.com/u",
"A &amp; B &lt;C&gt;",
"#a &amp; #b",
)),
false,
"<a href=\"https://x.com/u\">A &amp; B</a>: C &lt;D&gt; &amp; E",
&[],
);
assert!(report.contains("title: A & B <C>"), "{report}");
assert!(report.contains("author: A & B"), "{report}");
assert!(report.contains("tags: #a & #b"), "{report}");
assert!(
report.contains("caption: <a href=\"https://x.com/u\">A & B</a>: C <D> & E"),
"{report}"
);
for entity in ["&amp;", "&lt;", "&gt;"] {
assert!(!report.contains(entity), "unexpected {entity} in: {report}");
}
}
#[test]
fn test_parse_report_is_capped() {
// 200 media lines ≈ 8 KB, comfortably over the cap.