add refresh token if expire

.gitignore

@@ -3,4 +3,4 @@ __pycache__/
 cert/
 data/
 docker-compose.yml
-x.py
+utils/x.py

Dockerfile

@@ -7,7 +7,7 @@ ENV PYTHONUNBUFFERED=1
 
 RUN set -eux; \
 	apt-get update; \
-	apt-get install -y gosu; \
+	apt-get install -y git gosu; \
 	rm -rf /var/lib/apt/lists/*; \
 # verify that the binary works
 	gosu nobody true

common.py

@@ -1,9 +1,8 @@
-import logging
 import os
-import re
 
 try:
-    import uvloop, asyncio
+    import uvloop
+    import asyncio
 
     asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
 except ImportError:
@@ -12,6 +11,8 @@ except ImportError:
 BOT_TOKEN = os.getenv("BOT_TOKEN")
 ADMIN = [int(i) for i in os.getenv("BOT_ADMIN").split(",")]
 
+PIXIV_REFRESH_TOKEN = os.getenv("PIXIV_REFRESH_TOKEN")
+
 WEBHOOK = os.getenv("WEBHOOK", False)
 if WEBHOOK:
     WEBHOOK_LISTEN = os.getenv("WEBHOOK_LISTEN", "0.0.0.0")
@@ -20,17 +21,3 @@ if WEBHOOK:
     WEBHOOK_KEY = os.getenv("WEBHOOK_KEY", "cert/private.key")
     WEBHOOK_CERT = os.getenv("WEBHOOK_CERT", "cert/cert.pem")
     WEBHOOK_SECRET_TOKEN = os.getenv("WEBHOOK_SECRET_TOKEN")
-
-x_url_regex = re.compile(r"^(?:https?://)(?:www\.|mobile\.|)(?:x|twitter|fixvx|vxtwitter)\.com/(.+)/status/(\d+)")
-x_media_regex = re.compile(r"^(?:https?://)(pbs|video)\.twimg\.com/(.*)")
-x_tco_regex = re.compile(r"(?:https?://)t\.co/.+$", re.M)
-message_url_regex = re.compile(r"\[.+]", re.S)
-
-logging.basicConfig(
-    level=os.getenv("LOG_LEVEL", "WARNING"),
-    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
-)
-
-
-def get_logger(name: str) -> logging.Logger:
-    return logging.getLogger(name)

docker-compose.yml.example

@@ -1,12 +1,13 @@
 services:
   tgxmb: image: yoursfunny/telegram-twitter-media-bot: latest
     restart: always
-    ports:
-      - "8443:8443"
+#    ports:
+#      - "8443:8443"
     environment:
       LOCAL_USER_ID: '1000'
       BOT_TOKEN: ''
       BOT_ADMIN: ''
+      PIXIV_REFRESH_TOKEN: ''
       WEBHOOK: false
       WEBHOOK_LISTEN: '127.0.0.1'
       WEBHOOK_PORT: 8443

main.py

@@ -10,13 +10,17 @@ from telegram.ext import (ApplicationBuilder, CallbackQueryHandler, CommandHandl
                           InlineQueryHandler, MessageHandler, PicklePersistence, filters)
 
 import common
-from tweet import Telegram
+import utils.regex as regex
+from utils.logger import get_logger
+from utils.net import NetClient
+from utils.pixiv import ProcessPixiv
+from utils.telegram import Telegram
 
 if TYPE_CHECKING:
     from telegram import Chat, Message, Update
     from telegram.ext import Application, ContextTypes
 
-logger = common.get_logger(__name__)
+logger = get_logger(__name__)
 
 
 def send_action(action):
@@ -37,8 +41,9 @@ async def inline_query(update: Update, context: ContextTypes.DEFAULT_TYPE) -> No
         return
     logger.info(f"Query: {query}")
     async with Telegram(query) as tweet:
-        result = list(tweet.inline_query_generator)
-        await update.inline_query.answer(result)
+        if not tweet:
+            return
+        await update.inline_query.answer(tweet.inline_query_result())
 
 
 @send_action(ChatAction.UPLOAD_PHOTO)
@@ -46,12 +51,15 @@ async def url_media(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
     url = update.message.text
     logger.info(f"Receiving url: {url}")
     async with Telegram(url) as tweet:
-        media = list(tweet.message_media_generator)
+        media = tweet.message_media_result()
+        if not media:
+            await update.effective_message.reply_text("No media found or media type is not supported.")
+            return
         message_to_send = await update.effective_message.reply_media_group(
             media,
             caption=tweet.message_text,
             reply_to_message_id=update.message.message_id,
-        ) if not isinstance(tuple, media[0]) else await update.effective_message.reply_animation(
+        ) if not isinstance(media[0], tuple) else await update.effective_message.reply_animation(
             media[0][0],
             caption=tweet.message_text,
             reply_to_message_id=update.message.message_id,
@@ -105,7 +113,7 @@ async def edit_message(update: Update, context: ContextTypes.DEFAULT_TYPE) -> No
         ))
     else:
         update_text = html.escape(update.message.text)
-        match = common.message_url_regex.search(update_text)
+        match = regex.message_url.search(update_text)
         if match:
             match = match.span()
             update_text = update_text[:match[0]] + message_url.format(
@@ -204,7 +212,9 @@ async def post_init(application: Application) -> None:
     DESCRIPTION = "A bot to fetch tweets from Twitter."
     await application.bot.set_my_description(DESCRIPTION)
     await application.bot.set_my_short_description(DESCRIPTION)
-    Telegram.init_client()
+    NetClient.init_client()
+    if common.PIXIV_REFRESH_TOKEN:
+        await ProcessPixiv.init_client(common.PIXIV_REFRESH_TOKEN)
 
 
 async def post_stop(application: Application) -> None:
@@ -212,7 +222,7 @@ async def post_stop(application: Application) -> None:
 
 
 async def post_shutdown(application: Application) -> None:
-    await Telegram.close_client()
+    await NetClient.close_client()
 
 
 def main():
@@ -234,8 +244,9 @@ def main():
     user_filter.add_user_ids(common.ADMIN)
 
     handlers = [
-        InlineQueryHandler(inline_query, common.x_url_regex),
-        MessageHandler(filters.Regex(common.x_url_regex) & filters.ChatType.PRIVATE, url_media),
+        InlineQueryHandler(inline_query),
+        MessageHandler((filters.Regex(regex.x_url) | filters.Regex(regex.pixiv_url)) & filters.ChatType.PRIVATE,
+                       url_media),
         CommandHandler("set_forward_channel", cmd_set_forward_channel),
         CommandHandler("remove_forward_channel", cmd_remove_forward_channel),
         CommandHandler("edit_before_forward", cmd_edit_before_forward),

requirements.txt

@@ -1,3 +1,4 @@
 python-telegram-bot[webhooks]~=21.3
-httpx[http2]~=0.27.0
+httpx[http2]~=0.27
 uvloop~=0.19.0; sys_platform != 'win32'
+async-pixiv @ git+https://github.com/TheFunny/async-pixiv@main

types.py

@@ -1,19 +0,0 @@
-from typing import Type, TypedDict
-
-from telegram import InlineQueryResultMpeg4Gif, InlineQueryResultPhoto, InlineQueryResultVideo, InputMediaPhoto, \
-    InputMediaVideo
-
-TypeInlineQueryResult: Type[
-    InlineQueryResultMpeg4Gif | InlineQueryResultPhoto | InlineQueryResultVideo] = InlineQueryResultMpeg4Gif | InlineQueryResultPhoto | InlineQueryResultVideo
-InputMediaAnimation = tuple[str, bool]
-TypeMessageMediaResult: Type[
-    InputMediaPhoto | InputMediaVideo | InputMediaAnimation] = InputMediaPhoto | InputMediaVideo | InputMediaAnimation
-
-
-class TweetInfo(TypedDict):
-    tweetID: str
-    user_name: str
-    user_screen_name: str
-    text: str
-    media_extended: list[dict]
-    possibly_sensitive: bool

utils/logger.py

@@ -0,0 +1,11 @@
+import logging
+import os
+
+logging.basicConfig(
+    level=os.getenv("LOG_LEVEL", "WARNING"),
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+
+
+def get_logger(name: str) -> logging.Logger:
+    return logging.getLogger(name)

utils/net.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from httpx import AsyncClient
+
+
+def create_client() -> AsyncClient:
+    return AsyncClient(http2=True)
+
+
+async def close_client(_client: AsyncClient) -> None:
+    return await _client.aclose()
+
+
+async def fetch_json(_client: AsyncClient, url: str) -> dict:
+    response = await _client.get(url)
+    assert response.is_success, f"Failed to fetch {url}, status code {response.status_code}"
+    return response.json()
+
+
+class NetClient:
+    _httpx_client: AsyncClient
+
+    @classmethod
+    def init_client(cls) -> None:
+        cls._httpx_client = create_client()
+
+    @classmethod
+    async def close_client(cls) -> None:
+        await close_client(cls._httpx_client)
+
+    @classmethod
+    def get_client(cls) -> AsyncClient:
+        return cls._httpx_client
+
+    @classmethod
+    async def fetch_json(cls, url: str) -> dict:
+        return await fetch_json(cls._httpx_client, url)

utils/pixiv.py

@@ -0,0 +1,212 @@
+from __future__ import annotations
+
+import html
+from typing import Generator, Literal, TYPE_CHECKING
+from uuid import uuid4
+
+from async_pixiv import PixivClient
+from async_pixiv.error import ApiError
+from telegram import InlineQueryResultPhoto, InputMediaPhoto
+
+from utils.logger import get_logger
+
+if TYPE_CHECKING:
+    from async_pixiv.model.illust import Illust
+    from utils.types import TypeInlineQueryResult, TypeMessageMediaResult
+
+logger = get_logger(__name__)
+
+message_raw_text = """<a href="{url}">{text}</a> / <a href="{author_url}">{author}</a>
+{tags}
+"""
+
+
+class PixivMedia:
+    __slots__ = ('_url', '_thumb')
+
+    def __init__(self, url: str, thumb: str):
+        self._url: str = url
+        self._thumb: str = thumb
+
+    def __str__(self):
+        return f"PixivMedia(url={self.url}, thumb={self.thumb}, large={self.large})"
+
+    @property
+    def url(self) -> str:
+        return self._url
+
+    @property
+    def thumb(self) -> str:
+        return self._thumb
+
+    @property
+    def large(self) -> str:
+        url = self._url.replace("img-original", "img-master").removesuffix(".jpg").removesuffix(".png")
+        return url + "_master1200.jpg"
+
+
+class Pixiv:
+    __slots__ = ('_illust',)
+
+    def __init__(self, illust: Illust):
+        self._illust: Illust = illust
+
+    @property
+    def url(self) -> str:
+        return str(self._illust.link).rstrip('/')
+
+    @property
+    def type(self) -> Literal["illust", "manga", "ugoira"]:
+        return self._illust.type.value
+
+    @property
+    def title(self) -> str:
+        return self._illust.title
+
+    @property
+    def author(self) -> str:
+        return self._illust.user.name
+
+    @property
+    def author_url(self) -> str:
+        return str(self._illust.user.link).rstrip('/')
+
+    @property
+    def description(self) -> str:
+        return self._illust.caption
+
+    @property
+    def tags(self) -> list[str]:
+        return [tag.name for tag in self._illust.tags]
+
+    @property
+    def is_multiple_pages(self) -> bool:
+        return self._illust.page_count > 1
+
+    @property
+    def is_nsfw(self) -> bool:
+        return self._illust.is_nsfw
+
+    @property
+    def is_ai(self) -> bool:
+        return self._illust.ai_type.value == 2
+
+    @property
+    def images(self) -> list[PixivMedia]:
+        if self.is_multiple_pages:
+            return [
+                PixivMedia(
+                    url=page.image_urls.original,
+                    thumb=page.image_urls.medium
+                )
+                for page in self._illust.meta_pages
+            ]
+        else:
+            return [
+                PixivMedia(  # should observe if image_url.original is always None for single page
+                    url=self._illust.image_urls.original or self._illust.meta_single_page.original,
+                    thumb=self._illust.image_urls.medium
+                )
+            ]
+
+
+class ProcessPixiv:
+    _client: PixivClient
+
+    __slots__ = ('_url', '_illust')
+
+    @classmethod
+    async def init_client(cls, token: str) -> None:
+        cls._client = PixivClient()
+        await cls._client.login_with_token(token)
+        cls._token = token
+
+    @classmethod
+    async def close_client(cls) -> None:
+        await cls._client.close()
+
+    @classmethod
+    async def refresh_token(cls):
+        await cls._client.login_with_token(cls._token)
+
+    def __init__(self, url: str):
+        self._url: str = url
+
+    async def __aenter__(self):
+        self._illust = await self._fetch_illust()
+        return Pixiv(self._illust)
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        pass
+
+    async def _fetch_illust(self) -> Illust:
+        illust_id = self._parse_illust_id()
+        try:
+            return (await self._client.ILLUST.detail(illust_id)).illust
+        except ApiError:
+            await self.refresh_token()
+            return (await self._client.ILLUST.detail(illust_id)).illust  # TODO use retry here
+
+    def _parse_illust_id(self) -> int:
+        return int(self._url.split("/")[-1])
+
+
+class TelegramPixiv:
+    __slots__ = ('_url', '_pixiv')
+
+    def __init__(self, url: str):
+        self._url = url
+
+    async def __aenter__(self):
+        async with ProcessPixiv(self._url) as pixiv:
+            self._pixiv = pixiv
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        pass
+
+    def url(self) -> str:
+        return self._url
+
+    @property
+    def message_text(self) -> str:
+        pixiv = self._pixiv
+        return message_raw_text.format(
+            url=pixiv.url,
+            author_url=pixiv.author_url,
+            author=html.escape(pixiv.author),
+            text=html.escape(pixiv.title),
+            tags=html.escape(" ".join(f"#{name}" for name in pixiv.tags))
+        )
+
+    def inline_query_result(self) -> tuple[TypeInlineQueryResult, ...]:
+        return tuple(self.inline_query_generator())
+
+    def message_media_result(self) -> tuple[TypeMessageMediaResult, ...]:
+        return tuple(self.message_media_generator())
+
+    def inline_query_generator(self) -> Generator[TypeInlineQueryResult, None, None]:
+        pixiv = self._pixiv
+        for media in pixiv.images:
+            logger.info(str(media))
+            if pixiv.type in ("illust", "manga"):
+                yield InlineQueryResultPhoto(
+                    id=str(uuid4()),
+                    photo_url=media.large,
+                    thumbnail_url=media.thumb,
+                    caption=self.message_text
+                )
+            else:
+                yield
+
+    def message_media_generator(self) -> Generator[TypeMessageMediaResult, None, None]:
+        pixiv = self._pixiv
+        for media in pixiv.images:
+            logger.info(str(media))
+            if pixiv.type in ("illust", "manga"):
+                yield InputMediaPhoto(
+                    media=media.url,
+                    has_spoiler=pixiv.is_nsfw
+                )
+            else:
+                yield

utils/regex.py

@@ -0,0 +1,8 @@
+import re
+
+x_url = re.compile(r"^(?:https?://)?(?:www\.|mobile\.)?(?:x|twitter|fixvx|vxtwitter)\.com/(.+)/status/(\d+)")
+x_media_url = re.compile(r"^(?:https?://)(pbs|video)\.twimg\.com/(.*)")
+x_tco_url = re.compile(r"(?:https?://)t\.co/.+$", re.M)
+message_url = re.compile(r"\[.+]", re.S)
+
+pixiv_url = re.compile(r"^(?:https?://)?(?:www\.)?pixiv\.net/(?:en/)?artworks/(\d+)")

utils/telegram.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from common import PIXIV_REFRESH_TOKEN
+from .pixiv import TelegramPixiv
+from .regex import pixiv_url, x_url
+from .tweet import TelegramTweet
+
+
+class Telegram:
+    def __init__(self, url: str):
+        self._url = url
+
+    async def __aenter__(self):
+        if x_url.match(self._url):
+            async with TelegramTweet(self._url) as tweet:
+                return tweet
+        elif PIXIV_REFRESH_TOKEN and pixiv_url.match(self._url):
+            async with TelegramPixiv(self._url) as pixiv:
+                return pixiv
+        else:
+            return None
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        pass

utils/tweet.py

@@ -5,14 +5,15 @@ from functools import cached_property
 from typing import Generator, TYPE_CHECKING
 from uuid import uuid4
 
-from httpx import AsyncClient
 from telegram import (InlineQueryResultMpeg4Gif, InlineQueryResultPhoto, InlineQueryResultVideo, InputMediaPhoto,
                       InputMediaVideo)
 
-from common import get_logger, x_media_regex, x_tco_regex, x_url_regex
+from .logger import get_logger
+from .net import NetClient
+from .regex import x_media_url, x_tco_url, x_url
 
 if TYPE_CHECKING:
-    from types import TweetInfo, TypeInlineQueryResult, TypeMessageMediaResult
+    from .types import TweetInfo, TypeInlineQueryResult, TypeMessageMediaResult
 
 logger = get_logger(__name__)
 
@@ -25,21 +26,6 @@ message_raw_text = """{url}
 """
 
 
-def create_client() -> AsyncClient:
-    return AsyncClient(http2=True)
-
-
-async def close_client(_client: AsyncClient) -> None:
-    await _client.aclose()
-
-
-async def fetch_json(_client: AsyncClient, url: str) -> dict:
-    logger.info(f"Fetching {url}")
-    response = await _client.get(url)
-    assert response.is_success, f"Failed to fetch {url}, status code {response.status_code}"
-    return response.json()
-
-
 class TweetMedia:
     __slots__ = ('_url', '_thumb', '_type', '__dict__')
 
@@ -53,7 +39,7 @@ class TweetMedia:
 
     @cached_property
     def _uri(self) -> str | None:
-        if match := x_media_regex.match(self._url):
+        if match := x_media_url.match(self._url):
             return match.group(2).removesuffix('.jpg').removesuffix('.png')
         return None
 
@@ -135,10 +121,9 @@ class Tweet:
 
 
 class ProcessTweet:
-    __slots__ = ('_httpx_client', '_url', '_tweet')
+    __slots__ = ('_url', '_tweet')
 
-    def __init__(self, httpx_client: AsyncClient, url: str):
-        self._httpx_client: AsyncClient = httpx_client
+    def __init__(self, url: str):
         self._url: str = url
 
     async def __aenter__(self):
@@ -156,14 +141,14 @@ class ProcessTweet:
         pass
 
     async def _fetch_tweet(self) -> TweetInfo:
-        match = x_url_regex.match(self._url)
+        match = x_url.match(self._url)
         assert match, f"Invalid URL: {self._url}"
-        auther_id, tweet_id = match.group()
-        return await fetch_json(self._httpx_client, vx_api_url.format(auther_id, tweet_id))
+        auther_id, tweet_id = match.groups()
+        return await NetClient.fetch_json(vx_api_url.format(auther_id, tweet_id))
 
     @property
     def _tweet_text(self) -> str:
-        match = x_tco_regex.search(self._tweet['text'])
+        match = x_tco_url.search(self._tweet['text'])
         return self._tweet['text'][:match.start()].strip(" ") if match else self._tweet['text']
 
     @property
@@ -179,14 +164,13 @@ class ProcessTweet:
 
 
 class TelegramTweet:
-    __slots__ = ('_httpx_client', '_url', '_tweet', '__dict__')
+    __slots__ = ('_url', '_tweet', '__dict__')
 
-    def __init__(self, httpx_client: AsyncClient, url: str):
-        self._httpx_client: AsyncClient = httpx_client
+    def __init__(self, url: str):
         self._url: str = url
 
     async def __aenter__(self):
-        async with ProcessTweet(self._httpx_client, self._url) as tweet:
+        async with ProcessTweet(self._url) as tweet:
             self._tweet = tweet
         return self
 
@@ -207,7 +191,12 @@ class TelegramTweet:
             text=html.escape(tweet.text)
         )
 
-    @property
+    def inline_query_result(self) -> tuple[TypeInlineQueryResult, ...]:
+        return tuple(self.inline_query_generator())
+
+    def message_media_result(self) -> tuple[TypeMessageMediaResult, ...]:
+        return tuple(self.message_media_generator())
+
     def inline_query_generator(self) -> Generator[TypeInlineQueryResult, None, None]:
         tweet = self._tweet
         for tweet_media in tweet.media:
@@ -236,7 +225,6 @@ class TelegramTweet:
                     caption=self.message_text
                 )
 
-    @property
     def message_media_generator(self) -> Generator[TypeMessageMediaResult, None, None]:
         tweet = self._tweet
         for tweet_media in tweet.media:
@@ -260,23 +248,3 @@ class TelegramTweet:
                     has_spoiler=tweet.sensitive,
                     thumbnail=tweet_media.thumb
                 )
-
-
-class Telegram:
-    _httpx_client: AsyncClient
-
-    def __init__(self, url: str):
-        self._url = url
-
-    async def __aenter__(self):
-        if x_url_regex.match(self._url):
-            async with TelegramTweet(self._httpx_client, self._url) as tweet:
-                return tweet
-
-    @classmethod
-    def init_client(cls) -> None:
-        cls._httpx_client = create_client()
-
-    @classmethod
-    async def close_client(cls) -> None:
-        await close_client(cls._httpx_client)

utils/types.py

@@ -0,0 +1,17 @@
+from typing import TypedDict
+
+from telegram import InlineQueryResultMpeg4Gif, InlineQueryResultPhoto, InlineQueryResultVideo, InputMediaPhoto, \
+    InputMediaVideo
+
+TypeInlineQueryResult = InlineQueryResultMpeg4Gif | InlineQueryResultPhoto | InlineQueryResultVideo
+InputMediaAnimation = tuple[str, bool]
+TypeMessageMediaResult = InputMediaPhoto | InputMediaVideo | InputMediaAnimation
+
+
+class TweetInfo(TypedDict):
+    tweetID: str
+    user_name: str
+    user_screen_name: str
+    text: str
+    media_extended: list[dict]
+    possibly_sensitive: bool