use http2 in httpx

.dockerignore

@@ -27,5 +27,5 @@
 .python-version
 LICENSE
 README.md
-pers_data
+data/
 cert/

.github/workflows/docker.yml

@@ -4,6 +4,8 @@ on:
   push:
     tags:
       - v*
+    branches:
+      - master
 
 env:
   APP_NAME: telegram-twitter-media-bot
@@ -40,7 +42,6 @@ jobs:
         name: Build and push
         uses: docker/build-push-action@v5
         with:
-          context: .
           push: true
           build-args: |
             APP_NAME=${{ env.APP_NAME }}

.gitignore

@@ -1,6 +1,6 @@
 **/.idea
 __pycache__/
 cert/
-pers_data
+data/
 docker-compose.yml
 x.py

common.py

@@ -1,9 +1,16 @@
 import logging
-import re
 import os
+import re
+
+try:
+    import uvloop, asyncio
+
+    asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
+except ImportError:
+    uvloop = None
 
 BOT_TOKEN = os.getenv("BOT_TOKEN")
-ADMIN = os.getenv("ADMIN").split(",")
+ADMIN = [int(i) for i in os.getenv("BOT_ADMIN").split(",")]
 
 WEBHOOK = os.getenv("WEBHOOK", False)
 if WEBHOOK:
@@ -14,12 +21,16 @@ if WEBHOOK:
     WEBHOOK_CERT = os.getenv("WEBHOOK_CERT", "cert/cert.pem")
     WEBHOOK_SECRET_TOKEN = os.getenv("WEBHOOK_SECRET_TOKEN")
 
-x_url_regex = re.compile(r"^(?:https?://)(?:www\.|mobile\.|)(x|twitter)\.com/.+/status/(\d+)")
+x_url_regex = re.compile(r"^(?:https?://)(?:www\.|mobile\.|)(?:x|twitter|fixvx|vxtwitter)\.com/(.+)/status/(\d+)")
 x_media_regex = re.compile(r"^(?:https?://)(pbs|video)\.twimg\.com/(.*)")
 x_tco_regex = re.compile(r"(?:https?://)t\.co/.+$", re.M)
+message_url_regex = re.compile(r"\[.+]", re.S)
 
 logging.basicConfig(
     level=os.getenv("LOG_LEVEL", "WARNING"),
     format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
 )
-logger = logging.getLogger(__name__)
+
+
+def get_logger(name: str) -> logging.Logger:
+    return logging.getLogger(name)

docker-compose.yml.example

@@ -1,4 +1,3 @@
-version: '3.7'
 services:
   tgxmb: image: yoursfunny/telegram-twitter-media-bot: latest
     restart: always
@@ -16,6 +15,7 @@ services:
       WEBHOOK_CERT: './cert/cert.pem'
       WEBHOOK_SECRET_TOKEN: 'secret-token'
 #      LOG_LEVEL: 'WARNING'
-#    volumes:
+volumes:
+      - ./data: /app/data
 #      - ./cert:/app/cert
     container_name: tgxmb

main.py

@@ -1,77 +1,157 @@
-from aiohttp import ClientSession
-from telegram import Update, Chat
-from telegram.constants import ParseMode, ChatAction
-from telegram.ext import (
-    Application,
-    ApplicationBuilder,
-    ContextTypes,
-    Defaults,
-    filters,
-    InlineQueryHandler,
-    PicklePersistence,
-    MessageHandler, CommandHandler
-)
+import html
+from functools import wraps
+from typing import TYPE_CHECKING
+
+from telegram import InlineKeyboardButton, InlineKeyboardMarkup
+from telegram.constants import ChatAction, ChatType, ParseMode
+from telegram.ext import (ApplicationBuilder, CallbackQueryHandler, CommandHandler, Defaults,
+                          InlineQueryHandler, MessageHandler, PicklePersistence, filters)
 
 import common
 from tweet import TGTweet
 
+if TYPE_CHECKING:
+    from telegram import Chat, Message, Update
+    from telegram.ext import Application, ContextTypes
+
+logger = common.get_logger(__name__)
+
+
+def send_action(action):
+    def decorator(func):
+        @wraps(func)
+        async def command_func(update: Update, context: ContextTypes.DEFAULT_TYPE, *args, **kwargs):
+            await update.effective_chat.send_action(action)
+            return await func(update, context, *args, **kwargs)
+
+        return command_func
+
+    return decorator
+
 
 async def inline_query(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
     query = update.inline_query.query
     if query == "":
         return
-    async with TGTweet(context.bot_data['client'], query) as tweet:
+    logger.info(f"Query: {query}")
+    async with TGTweet(query) as tweet:
         result = list(tweet.inline_query_generator)
         await update.inline_query.answer(result)
 
 
-async def reply_media(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
-    await update.effective_chat.send_action(ChatAction.UPLOAD_PHOTO)
-    # url = update.message.text.split(" ")[0]
+@send_action(ChatAction.UPLOAD_PHOTO)
+async def url_media(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
     url = update.message.text
-    async with TGTweet(context.bot_data['client'], url) as tweet:
+    logger.info(f"Receiving url: {url}")
+    async with TGTweet(url) as tweet:
         media = list(tweet.pm_media_generator)
-        message_sent = await update.effective_message.reply_media_group(
-            media, caption=tweet.message_text,
-            # reply_to_message_id=update.message.message_id
+        message_to_send = await update.effective_message.reply_media_group(
+            media,
+            caption=tweet.message_text,
+            reply_to_message_id=update.message.message_id,
+        ) if not tweet.is_single_gif else await update.effective_message.reply_animation(
+            media[0][0],
+            caption=tweet.message_text,
+            reply_to_message_id=update.message.message_id,
+            has_spoiler=media[0][1]
         )
+        if not isinstance(message_to_send, tuple):
+            message_to_send = (message_to_send,)
+        url = tweet.url
+    if context.user_data.get('edit_before_forward', False):
+        message_reply = await update.effective_message.reply_text(
+            "Reply to edit message. [URL]",
+            reply_markup=InlineKeyboardMarkup.from_button(
+                InlineKeyboardButton("↩️ Confirm", callback_data="forward")
+            ),
+            reply_to_message_id=update.message.message_id,
+        )
+        context.user_data['message_reply'] = message_reply
+        context.user_data['message_to_send'] = message_to_send
+        context.user_data['message_url'] = url
+        return
     if 'forward_channel_id' in context.user_data:
-        try:
-            await update.effective_chat.copy_messages(
-                chat_id=context.user_data['forward_channel_id'],
-                message_ids=[m.id for m in message_sent],
-            )
-        except Exception as e:
-            await update.effective_message.reply_text(str(e))
+        await forward_message(update, context, message_to_send)
 
 
+async def forward_message(
+        update: Update,
+        context: ContextTypes.DEFAULT_TYPE,
+        message_to_send: tuple[Message, ...],
+) -> None:
+    try:
+        await update.effective_chat.copy_messages(
+            context.user_data['forward_channel_id'],
+            [m.id for m in message_to_send]
+        )
+    except Exception as e:
+        await update.effective_message.reply_text(str(e))
+
+
+async def edit_message(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
+    if 'message_reply' not in context.user_data:
+        return
+    if update.message.reply_to_message != context.user_data['message_reply']:
+        return
+    template = context.user_data.get('template', None)
+    message_url = '<a href="{0}">{1}</a>'
+    url = context.user_data['message_url']
+    if template:
+        update_text = template.replace("[]", message_url.format(
+            url,
+            html.escape(update.message.text)
+        ))
+    else:
+        update_text = html.escape(update.message.text)
+        match = common.message_url_regex.search(update_text)
+        if match:
+            match = match.span()
+            update_text = update_text[:match[0]] + message_url.format(
+                url,
+                update_text[match[0] + 1:match[1] - 1]
+            ) + update_text[match[1]:]
+    message_to_send = context.user_data['message_to_send']
+    await message_to_send[0].edit_caption(update_text)
+
+
+async def query_forward_message(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
+    message_to_send = context.user_data['message_to_send']
+    await forward_message(update, context, message_to_send)
+    await update.callback_query.answer('✅ Forwarded')
+    await update.callback_query.delete_message()
+    del context.user_data['message_reply']
+    del context.user_data['message_to_send']
+    del context.user_data['message_url']
+
+
+@send_action(ChatAction.TYPING)
 async def cmd_set_forward_channel(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
-    await update.effective_chat.send_action(ChatAction.TYPING)
     if not context.args:
         await update.effective_message.reply_text("Please provide a channel username or id.")
         return
     channel = context.args[0]
     try:
-        channel: Chat = context.bot.get_chat(channel)
+        channel: Chat = await context.bot.get_chat(channel)
     except Exception as e:
         await update.effective_message.reply_text(str(e))
         return
-    if channel.type != "CHANNEL":
+    if channel.type != ChatType.CHANNEL:
         await update.effective_message.reply_text("That is not a channel.")
         return
-    channel_admin = await channel.get_administrators()
+    try:
+        channel_admin = await channel.get_administrators()
+    except Exception as e:
+        await update.effective_message.reply_text(str(e) + "\nPlease add the bot to the channel and set as admin")
+        return
     user_bot = filter(lambda x: x.user.id == context.bot.id, channel_admin)
     user_bot = next(user_bot, None)
-    if not user_bot:
-        await update.effective_message.reply_text("I am not an admin in that channel.")
-        return
     if user_bot.can_post_messages:
         context.user_data['forward_channel_id'] = channel.id
         await update.effective_message.reply_text("Add successfully.")
 
 
+@send_action(ChatAction.TYPING)
 async def cmd_remove_forward_channel(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
-    await update.effective_chat.send_action(ChatAction.TYPING)
     if 'forward_channel_id' in context.user_data:
         del context.user_data['forward_channel_id']
         await update.effective_message.reply_text("Remove successfully.")
@@ -79,6 +159,41 @@ async def cmd_remove_forward_channel(update: Update, context: ContextTypes.DEFAU
     await update.effective_message.reply_text("No channel to remove.")
 
 
+@send_action(ChatAction.TYPING)
+async def cmd_edit_before_forward(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
+    if context.user_data.get('forward_channel_id', None) is None:
+        await update.effective_message.reply_text("Please enable forward channel first.")
+        return
+    ebf_status = context.user_data.get('edit_before_forward', False)
+    if ebf_status:
+        context.user_data['edit_before_forward'] = False
+        context.user_data.pop('message_reply', None)
+        context.user_data.pop('message_to_send', None)
+        context.user_data.pop('message_url', None)
+        await update.effective_message.reply_text("Disable edit before forward.")
+        return
+    context.user_data['edit_before_forward'] = True
+    await update.effective_message.reply_text("Enable edit before forward.")
+
+
+@send_action(ChatAction.TYPING)
+async def cmd_set_template(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
+    reply = update.effective_message.reply_to_message
+    if not reply:
+        await update.effective_message.reply_text("Please reply to a message to set as template.")
+        return
+    if '[]' not in reply.text_html:
+        await update.effective_message.reply_text("Please reply to a message with [] to set as template.")
+        return
+    context.user_data['template'] = reply.text_html
+    await update.effective_message.reply_text("Template set.")
+
+
+@send_action(ChatAction.TYPING)
+async def cmd_user_dict(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
+    await update.effective_message.reply_text(str(context.user_data))
+
+
 async def post_init(application: Application) -> None:
     # commands = [
     #     BotCommand('start', CMD_START),
@@ -87,7 +202,7 @@ async def post_init(application: Application) -> None:
     DESCRIPTION = "A bot to fetch tweets from Twitter."
     await application.bot.set_my_description(DESCRIPTION)
     await application.bot.set_my_short_description(DESCRIPTION)
-    application.bot_data['client'] = ClientSession()
+    TGTweet.init_client()
 
 
 async def post_stop(application: Application) -> None:
@@ -95,13 +210,12 @@ async def post_stop(application: Application) -> None:
 
 
 async def post_shutdown(application: Application) -> None:
-    await application.bot_data['client'].close()
+    await TGTweet.close_client()
 
 
 def main():
     defaults = Defaults(parse_mode=ParseMode.HTML, allow_sending_without_reply=True)
-    persistence = PicklePersistence(filepath='pers_data')
-    # proxy_url = 'http://127.0.0.1:10809'
+    persistence = PicklePersistence(filepath='data/pers.pkl')
     application = (ApplicationBuilder()
                    .token(common.BOT_TOKEN)
                    .defaults(defaults)
@@ -109,20 +223,23 @@ def main():
                    .post_init(post_init)
                    .post_stop(post_stop)
                    .post_shutdown(post_shutdown)
-                   # .proxy(proxy_url)
-                   # .get_updates_proxy(proxy_url)
+                   .concurrent_updates(True)
                    .build()
                    )
 
-    # user_filter = filters.User()
-    # user_filter.add_user_ids(common.admin)
+    user_filter = filters.User()
+    user_filter.add_user_ids(common.ADMIN)
 
     handlers = [
-        MessageHandler(filters.Regex(common.x_url_regex) & filters.ChatType.PRIVATE, reply_media),
-        # MessageHandler(None, reply_media),
         InlineQueryHandler(inline_query, common.x_url_regex),
+        MessageHandler(filters.Regex(common.x_url_regex) & filters.ChatType.PRIVATE, url_media),
         CommandHandler("set_forward_channel", cmd_set_forward_channel),
         CommandHandler("remove_forward_channel", cmd_remove_forward_channel),
+        CommandHandler("edit_before_forward", cmd_edit_before_forward),
+        CommandHandler("set_template", cmd_set_template),
+        MessageHandler(~filters.COMMAND & filters.ChatType.PRIVATE, edit_message),
+        CallbackQueryHandler(query_forward_message, pattern="forward"),
+        CommandHandler("bot_dict", cmd_user_dict, filters=user_filter),
     ]
 
     application.add_handlers(handlers)

requirements.txt

@@ -1,2 +1,3 @@
-python-telegram-bot[webhooks]~=21.0.1
-aiohttp[speedups]~=3.9.3
+python-telegram-bot[webhooks]~=21.3
+httpx[http2]~=0.27.0
+uvloop~=0.19.0; sys_platform != 'win32'

tweet.py

@@ -1,26 +1,36 @@
+import html
 from typing import Generator
 
-from aiohttp import ClientSession
-from telegram import (
-    InlineQueryResultPhoto,
-    InlineQueryResultVideo,
-    InputMediaPhoto,
-    InputMediaVideo
-)
-from common import x_url_regex, x_media_regex, x_tco_regex
+from httpx import AsyncClient
+from telegram import (InlineQueryResultMpeg4Gif, InlineQueryResultPhoto, InlineQueryResultVideo, InputMediaPhoto,
+                      InputMediaVideo)
+
+from common import get_logger, x_media_regex, x_tco_regex, x_url_regex
+
+logger = get_logger(__name__)
+
 
 twimg_url = 'https://pbs.twimg.com/'
-vx_api_url = 'https://api.vxtwitter.com/status/'
+vx_api_url = 'https://api.vxtwitter.com/{0}/status/{1}'
 
 message_raw_text = """{url}
 <a href="{author_url}">{author}</a>: {text}
 """
 
 
-async def fetch_json(session: ClientSession, url: str) -> dict:
-    async with session.get(url) as response:
-        assert response.status == 200
-        return await response.json()
+def create_client() -> 'AsyncClient':
+    return AsyncClient(http2=True)
+
+
+async def close_client(_client: 'AsyncClient') -> None:
+    await _client.aclose()
+
+
+async def fetch_json(_client: 'AsyncClient', url: str) -> dict:
+    logger.info(f"Fetching {url}")
+    response = await _client.get(url)
+    assert response.status_code == response.is_success, f"Failed to fetch {url}, status code {response.status_code}"
+    return response.json()
 
 
 class TweetMedia:
@@ -29,20 +39,27 @@ class TweetMedia:
         self._thumb: str = thumb
         self._type: str = media_type
 
+    def __str__(self):
+        return f"Media[url: {self.url} thumb: {self.thumb} type: {self.type}]"
+
     @property
     def _uri(self) -> str | None:
         match = x_media_regex.match(self._url)
         if match:
-            return match.group(2).removesuffix('.jpg')
+            return match.group(2).removesuffix('.jpg').removesuffix('.png')
         return None
 
     @property
     def url(self) -> str:
         match self._type:
             case "image":
-                return f"{twimg_url}{self._uri}?format=jpg&name=orig"
+                return f"{twimg_url}{self._uri}?format=jpg&name=4096x4096"
             case "video":
                 return self._url
+            case "gif":
+                return self._url
+            case _:
+                return self._url
 
     @property
     def thumb(self) -> str:
@@ -51,6 +68,10 @@ class TweetMedia:
                 return f"{twimg_url}{self._uri}?format=jpg&name=thumb"
             case "video":
                 return self._thumb
+            case "gif":
+                return self._thumb
+            case _:
+                return self._thumb
 
     @property
     def type(self) -> str:
@@ -104,28 +125,38 @@ class Tweet:
 
 
 class TGTweet(Tweet):
-    def __init__(self, session: ClientSession, url: str):
-        self._session: ClientSession = session
+    _httpx_client: AsyncClient
+
+    def __init__(self, url: str):
         self._url: str = url
-        self._id: str = self._tweet_id
-        assert self._id
+        self._api_param: tuple[str] = self._tweet_id
+        self._is_single_gif: bool = False
+        assert self._api_param
 
     async def __aenter__(self):
-        self._tweet: dict = await self._fetch_tweet(self._id)
+        self._tweet: dict = await self._fetch_tweet(self._api_param)
         super().__init__(*self._init_properties)
         return self
 
     async def __aexit__(self, exc_type, exc_val, exc_tb):
         pass
 
-    async def _fetch_tweet(self, tweet_id: str) -> dict:
-        return await fetch_json(self._session, vx_api_url + tweet_id)
+    @classmethod
+    def init_client(cls) -> None:
+        cls._httpx_client = create_client()
+
+    @classmethod
+    async def close_client(cls) -> None:
+        await close_client(cls._httpx_client)
+
+    async def _fetch_tweet(self, api_param: tuple[str]) -> dict:
+        return await fetch_json(self._httpx_client, vx_api_url.format(*api_param))
 
     @property
-    def _tweet_id(self) -> str | None:
+    def _tweet_id(self) -> tuple[str] | None:
         match = x_url_regex.match(self._url)
         if match:
-            return match.group(1).strip('/')
+            return match.groups()
         return None
 
     @property
@@ -146,48 +177,61 @@ class TGTweet(Tweet):
 
     @property
     def _init_properties(self) -> tuple:
+        id = self._tweet['tweetID']
         author = self._tweet['user_name']
         author_id = self._tweet['user_screen_name']
         text = self._tweet_text
         media = self._tweet_media
         sensitive = self._tweet['possibly_sensitive']
-        return self._id, author, author_id, text, media, sensitive
+        return id, author, author_id, text, media, sensitive
+
+    @property
+    def is_single_gif(self) -> bool:
+        return self._is_single_gif
 
     @property
     def message_text(self) -> str:
         return message_raw_text.format(
             url=self.url,
             author_url=self.author_url,
-            author=self.author,
-            text=self.text
+            author=html.escape(self.author),
+            text=html.escape(self.text)
         )
 
     @property
-    def inline_query_generator(self) -> Generator[InlineQueryResultPhoto | InlineQueryResultVideo, None, None]:
+    def inline_query_generator(self) -> Generator[
+        InlineQueryResultPhoto | InlineQueryResultVideo | InlineQueryResultMpeg4Gif, None, None
+    ]:
         for i, tweet_media in enumerate(self.media):
+            logger.info(str(tweet_media))
             if tweet_media.type == "image":
                 yield InlineQueryResultPhoto(
-                    id=self.id,
+                    id=str(i),
                     photo_url=tweet_media.url,
                     thumbnail_url=tweet_media.thumb,
-                    title=self.url,
-                    description=self.text,
-                    caption=self.message_text if not i else None
+                    caption=self.message_text
                 )
             elif tweet_media.type == "video":
                 yield InlineQueryResultVideo(
-                    id=self.id,
+                    id=str(i),
                     video_url=tweet_media.url,
                     mime_type="video/mp4",
                     thumbnail_url=tweet_media.thumb,
-                    title=self.url,
-                    description=self.text,
-                    caption=self.message_text if not i else None
+                    title=self.text,
+                    caption=self.message_text
+                )
+            elif tweet_media.type == "gif":
+                yield InlineQueryResultMpeg4Gif(
+                    id=str(i),
+                    mpeg4_url=tweet_media.url,
+                    thumbnail_url=tweet_media.thumb,
+                    caption=self.message_text
                 )
 
     @property
-    def pm_media_generator(self) -> Generator[InputMediaPhoto | InputMediaVideo, None, None]:
+    def pm_media_generator(self) -> Generator[InputMediaPhoto | InputMediaVideo | tuple[str, bool], None, None]:
         for tweet_media in self.media:
+            logger.info(str(tweet_media))
             if tweet_media.type == "image":
                 yield InputMediaPhoto(
                     media=tweet_media.url,
@@ -199,3 +243,12 @@ class TGTweet(Tweet):
                     has_spoiler=self.sensitive,
                     thumbnail=tweet_media.thumb
                 )
+            elif tweet_media.type == "gif":
+                if len(self.media) == 1:
+                    self._is_single_gif = True
+                    yield tweet_media.url, self.sensitive
+                yield InputMediaVideo(
+                    media=tweet_media.url,
+                    has_spoiler=self.sensitive,
+                    thumbnail=tweet_media.thumb
+                )