sync: update from private repo (ddadfd71)

scripts/x-adapter/test_x_adapter.py

@@ -29,6 +29,7 @@ def ns(**kw):
 check("ref: bare id", xa.parse_tweet_ref("20") == "20")
 check("ref: x.com url", xa.parse_tweet_ref("https://x.com/jack/status/20") == "20")
 check("ref: twitter.com url", xa.parse_tweet_ref("https://twitter.com/jack/statuses/20?s=1") == "20")
+check("ref: article url", xa.parse_tweet_ref("https://x.com/jack/article/2072471529242407210") == "2072471529242407210")
 check("ref: junk -> None", xa.parse_tweet_ref("not a tweet") is None)
 check("ref: empty -> None", xa.parse_tweet_ref("") is None)
 
@@ -162,6 +163,103 @@ check("home: who-to-follow user skipped", xa.parse_home_timeline(
             {"entryId": "cursor-top", "content": {"cursorType": "Top"}},
         ]}]}}}}, limit=10) == [])
 
+# ── find_tweet_result_in_detail / extract_article (X 長文記事) ──
+# TweetDetail レスポンスの実構造を模した fixture (2026-07 実データの縮約)
+article_result = {
+    "rest_id": "500",
+    "legacy": {"full_text": "https://t.co/xxxx", "created_at": "Wed Oct 10 20:19:24 +0000 2018",
+               "favorite_count": 1, "retweet_count": 0, "reply_count": 0},
+    "article": {"article_results": {"result": {
+        "rest_id": "499",
+        "title": "記事タイトル",
+        "preview_text": "プレビュー文",
+        "plain_text": "本文です。" * 10,
+        "metadata": {"first_published_at_secs": 1782950691},
+        "cover_media": {"media_info": {"original_img_url": "https://pbs.twimg.com/media/COVER.jpg"}},
+    }}},
+}
+detail_payload = {"data": {"threaded_conversation_with_injections_v2": {"instructions": [
+    {"type": "TimelineAddEntries", "entries": [
+        {"entryId": "tweet-1", "content": {"itemContent": {"tweet_results": {"result": {"rest_id": "1", "legacy": {}}}}}},
+        {"entryId": "tweet-500", "content": {"itemContent": {"tweet_results": {"result": article_result}}}},
+    ]},
+]}}}
+found = xa.find_tweet_result_in_detail(detail_payload, "500")
+check("detail: focal tweet found", found is not None and found.get("rest_id") == "500")
+check("detail: missing id -> None", xa.find_tweet_result_in_detail(detail_payload, "999") is None)
+check("detail: empty payload -> None", xa.find_tweet_result_in_detail({}, "500") is None)
+
+# visibility wrapper 越しでも見つかる
+wrapped_payload = {"data": {"entries": [{"tweet_results": {"result": {
+    "__typename": "TweetWithVisibilityResults", "tweet": article_result}}}]}
+}
+found_w = xa.find_tweet_result_in_detail(wrapped_payload, "500")
+check("detail: visibility wrapper unwrapped", found_w is not None and found_w.get("rest_id") == "500")
+
+art = xa.extract_article(article_result)
+check("article: title", art["title"] == "記事タイトル")
+check("article: previewText", art["previewText"] == "プレビュー文")
+check("article: plainText", art["plainText"] == "本文です。" * 10)
+check("article: publishedAtISO", art["publishedAtISO"] == "2026-07-02T00:04:51+00:00")
+check("article: coverImageUrl", art["coverImageUrl"] == "https://pbs.twimg.com/media/COVER.jpg")
+check("article: not truncated", "plainTextTruncated" not in art)
+
+# cap: full_text=False では ARTICLE_TEXT_CAP で切る
+long_art = {"article": {"article_results": {"result": {
+    "title": "t", "plain_text": "あ" * (xa.ARTICLE_TEXT_CAP + 100)}}}}
+capped = xa.extract_article(long_art)
+check("article: capped length", len(capped["plainText"]) == xa.ARTICLE_TEXT_CAP)
+check("article: truncated flag", capped.get("plainTextTruncated") is True)
+full = xa.extract_article(long_art, full_text=True)
+check("article: full_text lifts cap", len(full["plainText"]) == xa.ARTICLE_TEXT_CAP + 100)
+check("article: full not truncated", "plainTextTruncated" not in full)
+
+# 記事なしツイート → None / 壊れた形 → None
+check("article: non-article -> None", xa.extract_article(result_legacy) is None)
+check("article: junk -> None", xa.extract_article({"article": {"article_results": {"result": "?"}}}) is None)
+check("article: non-dict -> None", xa.extract_article(None) is None)
+
+# plain_text 無し (fieldToggles 未対応時) でも title/preview は返す
+no_body = {"article": {"article_results": {"result": {"title": "t2", "preview_text": "p2"}}}}
+nb = xa.extract_article(no_body)
+check("article: no body still returns meta", nb["title"] == "t2" and "plainText" not in nb)
+
+# ── extract_article_media (記事内の埋め込み画像・動画) ──
+img_entity = {"media_id": "111", "media_info": {
+    "__typename": "ApiImage",
+    "original_img_url": "https://pbs.twimg.com/media/IMG1.jpg",
+    "original_img_width": 1983, "original_img_height": 793}}
+vid_entity = {"media_id": "222", "media_info": {
+    "__typename": "ApiVideo",
+    "duration_millis": 7658,
+    "preview_image": {"original_img_url": "https://pbs.twimg.com/amplify_video_thumb/222/img/P.jpg"},
+    "variants": [
+        {"bit_rate": 2176000, "content_type": "video/mp4", "url": "https://video.twimg.com/a/720.mp4"},
+        {"content_type": "application/x-mpegURL", "url": "https://video.twimg.com/a/pl.m3u8"},
+    ]}}
+gif_entity = {"media_id": "333", "media_info": {
+    "__typename": "ApiGif",
+    "preview_image": {"original_img_url": "https://pbs.twimg.com/tweet_video_thumb/G.jpg"},
+    "variants": [{"content_type": "video/mp4", "url": "https://video.twimg.com/tweet_video/g.mp4"}]}}
+unknown_entity = {"media_id": "444", "media_info": {"__typename": "ApiAudioSpace"}}
+art_with_media = {"article": {"article_results": {"result": {
+    "title": "t", "media_entities": [img_entity, vid_entity, gif_entity, unknown_entity]}}}}
+am = xa.extract_article_media(art_with_media)
+check("amedia: count (unknown skipped)", len(am) == 3)
+check("amedia: photo", am[0] == {"type": "photo", "url": "https://pbs.twimg.com/media/IMG1.jpg"})
+check("amedia: video poster", am[1]["type"] == "video" and am[1]["url"].endswith("P.jpg"))
+check("amedia: video variant mapped", am[1]["variants"][0] ==
+      {"url": "https://video.twimg.com/a/720.mp4", "bitrate": 2176000, "contentType": "video/mp4"})
+check("amedia: m3u8 variant kept", am[1]["variants"][1]["contentType"] == "application/x-mpegURL")
+check("amedia: gif -> animated_gif", am[2]["type"] == "animated_gif" and am[2]["variants"][0]["url"].endswith("g.mp4"))
+check("amedia: non-article -> []", xa.extract_article_media(result_legacy) == [])
+check("amedia: no media_entities -> []", xa.extract_article_media(no_body) == [])
+check("amedia: non-dict -> []", xa.extract_article_media(None) == [])
+
+# 件数 cap: ARTICLE_MEDIA_CAP を超えたら切る
+many = {"article": {"article_results": {"result": {"media_entities": [img_entity] * (xa.ARTICLE_MEDIA_CAP + 5)}}}}
+check("amedia: capped", len(xa.extract_article_media(many)) == xa.ARTICLE_MEDIA_CAP)
+
 # ── emit (optional, needs PyYAML) ──
 try:
     import io

scripts/x-adapter/x_adapter.py

@@ -27,6 +27,8 @@
     - id, text, author{name,screenName,profileImageUrl,id},
       metrics{likes,retweets,replies,views}, createdAtISO, isRetweet,
       media[]{type, url, variants[]{url,bitrate,contentType}}
+      article{title, previewText, plainText, ...}   # X 長文記事のみ (tweet サブコマンド)
+      # 長文記事の埋め込み画像・動画は media[] に合流する (自動 DL 対象)
 
 注意: twscrape / twikit / httpx は遅延 import (関数内)。pure なマッピング関数だけ
 なら依存なしで import でき、test_x_adapter.py がネットワーク・依存なしで回る。
@@ -39,7 +41,7 @@ import re
 import sys
 from email.utils import parsedate_to_datetime
 
-VERSION = "x-adapter 1.0.0 (twscrape backend)"
+VERSION = "x-adapter 1.2.0 (twscrape backend)"
 
 # twitter-cli の YAML スキーマバージョン (x.ts は使わないが互換のため踏襲)
 SCHEMA_VERSION = "1"
@@ -48,13 +50,17 @@ SCHEMA_VERSION = "1"
 # ── pure helpers (依存なし・テスト対象) ──────────────────────────────
 
 def parse_tweet_ref(raw: str) -> str | None:
-    """tweet ID もしくは status URL から数値 ID を取り出す。"""
+    """tweet ID もしくは status / article URL から数値 ID を取り出す。
+
+    X の長文記事 (X Articles) は `x.com/{user}/article/{id}` 形式で共有されるが、
+    その id はツイート ID と同じ空間なので TweetDetail でそのまま引ける。
+    """
     s = (raw or "").strip()
     if not s:
         return None
     if s.isdigit():
         return s
-    m = re.search(r"(?:x\.com|twitter\.com)/[^/]+/status(?:es)?/(\d+)", s)
+    m = re.search(r"(?:x\.com|twitter\.com)/[^/]+/(?:status(?:es)?|article)/(\d+)", s)
     return m.group(1) if m else None
 
 
@@ -219,6 +225,113 @@ def parse_graphql_tweet_result(result: dict) -> dict | None:
     }
 
 
+# ── X 長文記事 (X Articles) の抽出 ──────────────────────────────────
+#
+# 長文記事ポストの legacy.full_text は記事への t.co リンク 1 本だけで本文を含まない。
+# 本文は TweetDetail レスポンスの tweet result 直下 `article.article_results.result`
+# に入る (plain_text は fieldToggles.withArticlePlainText: true を付けた時のみ)。
+
+# plain_text の既定上限 (文字数)。LLM コンテキスト保護。--full-text で解除。
+ARTICLE_TEXT_CAP = 12000
+
+# 記事内埋め込みメディアの件数上限。x.ts の downloadTweetMedia は media[] を
+# 全件 DL する (件数制限なし・サイズ上限のみ) ので、画像だらけの記事で
+# ダウンロードが暴発しないようアダプタ側で頭を抑える。
+ARTICLE_MEDIA_CAP = 20
+
+
+def find_tweet_result_in_detail(payload, tid: str) -> dict | None:
+    """TweetDetail レスポンス全体から rest_id == tid の tweet_results.result を探す。
+
+    conversation スレッド内の別ツイート (リプライ等) も同じ形で並ぶので、
+    rest_id の一致で focal tweet を特定する。visibility wrapper は剥がして返す。
+    """
+    if isinstance(payload, dict):
+        tr = payload.get("tweet_results")
+        if isinstance(tr, dict):
+            r = tr.get("result")
+            if isinstance(r, dict):
+                if r.get("__typename") == "TweetWithVisibilityResults":
+                    r = r.get("tweet") if isinstance(r.get("tweet"), dict) else r
+                if isinstance(r, dict) and str(r.get("rest_id") or "") == str(tid):
+                    return r
+        for v in payload.values():
+            hit = find_tweet_result_in_detail(v, tid)
+            if hit is not None:
+                return hit
+    elif isinstance(payload, list):
+        for v in payload:
+            hit = find_tweet_result_in_detail(v, tid)
+            if hit is not None:
+                return hit
+    return None
+
+
+def extract_article(result, full_text: bool = False) -> dict | None:
+    """tweet result の article.article_results.result から記事情報を YAML dict に。
+
+    記事でないツイート・形が崩れている場合は None (呼び出し側は無視して従来出力)。
+    """
+    art = _dig(result if isinstance(result, dict) else {}, "article", "article_results", "result")
+    if not isinstance(art, dict):
+        return None
+    out: dict = {
+        "title": art.get("title") or "",
+        "previewText": art.get("preview_text") or "",
+    }
+    plain = art.get("plain_text")
+    if isinstance(plain, str) and plain:
+        if not full_text and len(plain) > ARTICLE_TEXT_CAP:
+            out["plainText"] = plain[:ARTICLE_TEXT_CAP]
+            out["plainTextTruncated"] = True
+        else:
+            out["plainText"] = plain
+    pub = _dig(art, "metadata", "first_published_at_secs")
+    if isinstance(pub, (int, float)) and pub > 0:
+        from datetime import datetime, timezone
+        out["publishedAtISO"] = datetime.fromtimestamp(int(pub), tz=timezone.utc).isoformat()
+    cover = _dig(art, "cover_media", "media_info", "original_img_url")
+    if cover:
+        out["coverImageUrl"] = cover
+    return out
+
+
+def extract_article_media(result) -> list:
+    """記事内の埋め込みメディア (media_entities) を row の media[] と同じ形に変換。
+
+    media_entities は fieldToggles.withArticleRichContentState: true の時のみ返る。
+    row["media"] に合流させることで x.ts の downloadTweetMedia がそのまま
+    自動 DL し localPath を付ける (記事ポストの tweet-level media は常に空なので
+    衝突しない)。未知の __typename (音声スペース等) は黙ってスキップ。
+    """
+    art = _dig(result if isinstance(result, dict) else {}, "article", "article_results", "result")
+    if not isinstance(art, dict):
+        return []
+    out: list = []
+    for entity in art.get("media_entities") or []:
+        if len(out) >= ARTICLE_MEDIA_CAP:
+            break
+        info = _dig(_as_dict(entity), "media_info") or {}
+        tname = info.get("__typename")
+        if tname == "ApiImage":
+            url = info.get("original_img_url")
+            if url:
+                out.append({"type": "photo", "url": url})
+        elif tname in ("ApiVideo", "ApiGif"):
+            variants = [
+                {"url": v.get("url"), "bitrate": v.get("bit_rate") or 0,
+                 "contentType": v.get("content_type") or ""}
+                for v in (info.get("variants") or [])
+                if isinstance(v, dict) and v.get("url")
+            ]
+            out.append({
+                "type": "video" if tname == "ApiVideo" else "animated_gif",
+                "url": _dig(info, "preview_image", "original_img_url") or "",
+                "variants": variants,
+            })
+    return out
+
+
 def _collect_tweet_results(node, out: list, limit: int) -> None:
     """entry / module item から tweet_results.result を拾って out に積む。
 
@@ -405,17 +518,77 @@ async def cmd_user_posts(args) -> None:
         shutil.rmtree(tmpdir, ignore_errors=True)
 
 
+async def _tweet_detail_raw_with_article(api, tid: str):
+    """TweetDetail を fieldToggles.withArticlePlainText 付きで直接叩く。
+
+    twscrape の tweet_details_raw は fieldToggles を渡せない (_gql_item が
+    variables/features のみ) ので、同じ op / variables / QueueClient を使い
+    パラメータだけ足す。X 長文記事の本文 (plain_text) は withArticlePlainText、
+    記事内埋め込みメディア (media_entities) は withArticleRichContentState が
+    無いとレスポンスに含まれない (後者は content_state も同乗するが emit しない)。
+    """
+    from twscrape.api import GQL_URL, OP_TweetDetail
+    from twscrape.api import GQL_FEATURES  # type: ignore[attr-defined]
+    from twscrape.queue_client import QueueClient
+    from twscrape.utils import encode_params
+
+    kv = {
+        "focalTweetId": str(tid),
+        "with_rux_injections": True,
+        "includePromotedContent": True,
+        "withCommunity": True,
+        "withQuickPromoteEligibilityTweetFields": True,
+        "withBirdwatchNotes": True,
+        "withVoice": True,
+        "withV2Timeline": True,
+    }
+    params = {
+        "variables": kv,
+        "features": GQL_FEATURES,
+        "fieldToggles": {"withArticlePlainText": True, "withArticleRichContentState": True},
+    }
+    queue = OP_TweetDetail.split("/")[-1]
+    async with QueueClient(api.pool, queue, False) as client:
+        return await client.get(f"{GQL_URL}/{OP_TweetDetail}", params=encode_params(params))
+
+
 async def cmd_tweet(args) -> None:
     tid = parse_tweet_ref(args.tweet)
     if not tid:
         fail(f"x-adapter: could not parse tweet id from '{args.tweet}'")
     api, tmpdir, shutil = await _make_api()
     try:
-        t = await api.tweet_details(int(tid))
+        # 記事本文トグル付きの直接呼び出しを優先。twscrape 内部 API の変化などで
+        # 壊れたら従来の tweet_details に落として XPostDetail 自体は生かす。
+        t = None
+        article = None
+        article_media: list = []
+        try:
+            from twscrape.models import parse_tweet
+            rep = await _tweet_detail_raw_with_article(api, tid)
+            if rep is not None:
+                t = parse_tweet(rep, int(tid))
+                result = find_tweet_result_in_detail(rep.json(), tid)
+                article = extract_article(result, full_text=bool(args.full_text))
+                article_media = extract_article_media(result)
+        except SystemExit:
+            raise
+        except Exception as e:
+            print(f"x-adapter: article-aware fetch failed, falling back: {type(e).__name__}: {e}",
+                  file=sys.stderr)
+        if not t:
+            t = await api.tweet_details(int(tid))
         if not t:
             await _ensure_alive(api)  # 認証切れなら nonzero
             fail(f"x-adapter: tweet {tid} not found or inaccessible")
-        emit([twscrape_tweet_to_dict(t)])
+        row = twscrape_tweet_to_dict(t)
+        if article:
+            row["article"] = article
+        if article_media:
+            # 記事ポストの tweet-level media は空なので合流しても衝突しない。
+            # x.ts の downloadTweetMedia がここを見て自動 DL → localPath 付与する。
+            row["media"] = (row.get("media") or []) + article_media
+        emit([row])
     finally:
         shutil.rmtree(tmpdir, ignore_errors=True)