fix: expand twitter short links like FxEmbed linkFixer

Replace display_text_range slicing with FxEmbed-style content matching:
expand mapped t.co links to their real URLs (dropping internal
x.com/i/web/status pages), then strip every leftover t.co short link
(appended media link, unmapped links).

The old code cut by display_text_range, whose index unit differs per
endpoint (UTF-16 on the syndication endpoint, code points in the
GraphQL fallback), so slicing by either unit left a partial
"https://t." caption tail on the other path. Content matching is
unit-agnostic and also keeps user-posted/quote links at the end of
the text that the trailing cut previously dropped.
This commit is contained in:
2026-08-07 10:25:54 +08:00
parent fb43441c56
commit 4060a88031
3 changed files with 98 additions and 63 deletions
+5 -6
View File
@@ -228,7 +228,6 @@ fn to_syndication_shape(tweet: &Value) -> Option<Value> {
"screen_name": user.get("screen_name"),
},
"possibly_sensitive": legacy.get("possibly_sensitive"),
"display_text_range": legacy.get("display_text_range"),
"entities": legacy.get("entities"),
"mediaDetails": legacy.pointer("/extended_entities/media"),
}))
@@ -251,12 +250,12 @@ mod tests {
"legacy": {
"id_str": "2083868672721039569",
"full_text": "nsfw content https://t.co/abc123",
"display_text_range": [0, 12],
"possibly_sensitive": true,
"entities": {
"urls": [
{ "url": "https://t.co/abc123", "expanded_url": "https://example.com/x" }
]
// The appended media link lives in extended_entities.media,
// not entities.urls, so it has no expansion mapping and the
// content-based strip removes it.
"urls": []
},
"extended_entities": {
"media": [
@@ -319,7 +318,7 @@ mod tests {
other => panic!("expected video, got {other:?}"),
}
assert_eq!(fetched.source_url, "https://x.com/nsfw_author/status/2083868672721039569");
// display_text_range cuts the trailing t.co link.
// The appended media short link (no URL-entity mapping) is stripped.
assert_eq!(fetched.title, "nsfw content");
}