This commit is contained in:
Executable
+81
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env bash
|
||||
# x-adapter インストーラ。
|
||||
#
|
||||
# 何をするか:
|
||||
# X.com の x-web バンドル移行で twitter-cli の transaction-id 生成が壊れたため、
|
||||
# twscrape (新バンドル対応の XClIdGen) + twikit を使う twitter-cli 互換アダプタを
|
||||
# 隔離 venv に入れ、`x-adapter` という実行ファイルを ~/.local/bin に置く。
|
||||
#
|
||||
# 使い方:
|
||||
# ./scripts/install-x-adapter.sh # 新規インストール
|
||||
# ./scripts/install-x-adapter.sh --upgrade # twscrape/twikit を最新へ (X が再破壊した時の修復レバー)
|
||||
#
|
||||
# 終わったら config.yaml に:
|
||||
# tools:
|
||||
# x_cli_command: [x-adapter]
|
||||
set -euo pipefail
|
||||
|
||||
MODE="install"
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--upgrade) MODE="upgrade" ;;
|
||||
*) echo "Usage: ./scripts/install-x-adapter.sh [--upgrade]" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
SRC="$SCRIPT_DIR/x-adapter/x_adapter.py"
|
||||
REQ="$SCRIPT_DIR/x-adapter/requirements.txt"
|
||||
|
||||
PREFIX="${X_ADAPTER_PREFIX:-$HOME/.local/share/x-adapter}"
|
||||
BIN_DIR="${X_ADAPTER_BIN:-$HOME/.local/bin}"
|
||||
VENV="$PREFIX/venv"
|
||||
|
||||
if [ ! -f "$SRC" ]; then
|
||||
echo "ERROR: $SRC not found." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$PREFIX" "$BIN_DIR"
|
||||
|
||||
if [ ! -d "$VENV" ]; then
|
||||
echo "Creating venv at $VENV ..."
|
||||
python3 -m venv "$VENV"
|
||||
fi
|
||||
|
||||
echo "Installing/upgrading dependencies ..."
|
||||
"$VENV/bin/pip" install -q --upgrade pip >/dev/null
|
||||
if [ "$MODE" = "upgrade" ]; then
|
||||
"$VENV/bin/pip" install -q --upgrade twscrape twikit httpx PyYAML
|
||||
else
|
||||
"$VENV/bin/pip" install -q -r "$REQ"
|
||||
fi
|
||||
|
||||
echo "Installing x_adapter.py ..."
|
||||
cp "$SRC" "$PREFIX/x_adapter.py"
|
||||
|
||||
WRAPPER="$BIN_DIR/x-adapter"
|
||||
cat > "$WRAPPER" <<EOF
|
||||
#!/usr/bin/env bash
|
||||
exec "$VENV/bin/python" "$PREFIX/x_adapter.py" "\$@"
|
||||
EOF
|
||||
chmod +x "$WRAPPER"
|
||||
|
||||
echo ""
|
||||
echo "x-adapter ready: $("$WRAPPER" --version 2>/dev/null || echo 'version unknown')"
|
||||
|
||||
echo ""
|
||||
echo "Smoke test (x-adapter --version) ..."
|
||||
if "$WRAPPER" --version >/dev/null 2>&1; then
|
||||
echo "Smoke test passed."
|
||||
else
|
||||
echo "WARN: x-adapter --version failed." >&2
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Next:"
|
||||
echo " 1. Ensure cookies are set in config.yaml (tools.x_auth_token / tools.x_ct0)"
|
||||
echo " 2. Set tools.x_cli_command: [x-adapter] (or [$WRAPPER] if ~/.local/bin is not on PATH)"
|
||||
echo " 3. Restart the server. XSearch / XUserPosts / XPostDetail / XTimeline now route through x-adapter."
|
||||
echo ""
|
||||
echo "If X breaks scraping again (every 2-4 weeks): ./scripts/install-x-adapter.sh --upgrade"
|
||||
@@ -0,0 +1,9 @@
|
||||
# x-adapter runtime deps.
|
||||
# twscrape: X の新 x-web バンドル向け x-client-transaction-id 生成 (XClIdGen) + 検索/ユーザー投稿/詳細
|
||||
# twikit: ホームタイムラインの GraphQL endpoint / FEATURES 定義の供給元
|
||||
# httpx: home timeline の raw GraphQL POST
|
||||
# PyYAML: twitter-cli 互換 YAML の出力
|
||||
twscrape>=0.19.0
|
||||
twikit>=2.3.3
|
||||
httpx>=0.27
|
||||
PyYAML>=6.0
|
||||
@@ -0,0 +1,187 @@
|
||||
#!/usr/bin/env python3
|
||||
"""x_adapter の pure マッピング関数の単体テスト。
|
||||
|
||||
ネットワーク・twscrape/twikit/httpx 無しで回る (それらは x_adapter 内で遅延 import)。
|
||||
実行: python3 scripts/x-adapter/test_x_adapter.py
|
||||
CI/手動どちらでも。失敗で exit 1。
|
||||
"""
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import x_adapter as xa # noqa: E402
|
||||
|
||||
_fails = []
|
||||
|
||||
|
||||
def check(name, cond):
|
||||
print(("PASS" if cond else "FAIL"), name)
|
||||
if not cond:
|
||||
_fails.append(name)
|
||||
|
||||
|
||||
def ns(**kw):
|
||||
return types.SimpleNamespace(**kw)
|
||||
|
||||
|
||||
# ── parse_tweet_ref ──
|
||||
check("ref: bare id", xa.parse_tweet_ref("20") == "20")
|
||||
check("ref: x.com url", xa.parse_tweet_ref("https://x.com/jack/status/20") == "20")
|
||||
check("ref: twitter.com url", xa.parse_tweet_ref("https://twitter.com/jack/statuses/20?s=1") == "20")
|
||||
check("ref: junk -> None", xa.parse_tweet_ref("not a tweet") is None)
|
||||
check("ref: empty -> None", xa.parse_tweet_ref("") is None)
|
||||
|
||||
# ── twitter_time_iso ──
|
||||
iso = xa.twitter_time_iso("Wed Oct 10 20:19:24 +0000 2018")
|
||||
check("time: iso year", iso.startswith("2018-10-10T20:19:24"))
|
||||
check("time: empty", xa.twitter_time_iso("") == "")
|
||||
check("time: garbage passthrough", xa.twitter_time_iso("xx") == "xx")
|
||||
|
||||
# ── twscrape_tweet_to_dict ──
|
||||
photo = ns(url="https://pbs.twimg.com/media/AAA.jpg")
|
||||
video = ns(thumbnailUrl="https://pbs.twimg.com/poster.jpg",
|
||||
variants=[ns(url="https://video.twimg.com/x.mp4", bitrate=832000, contentType="video/mp4")])
|
||||
media = ns(photos=[photo], videos=[video], animated=[])
|
||||
user = ns(id_str="12", displayname="Jack", username="jack",
|
||||
profileImageUrl="https://pbs.twimg.com/pp.jpg")
|
||||
tw = ns(id_str="20", id=20, rawContent="hello", user=user,
|
||||
likeCount=5, retweetCount=2, replyCount=1, viewCount=99,
|
||||
date=ns(isoformat=lambda: "2018-10-10T20:19:24+00:00"),
|
||||
retweetedTweet=None, media=media)
|
||||
d = xa.twscrape_tweet_to_dict(tw)
|
||||
check("tw: id", d["id"] == "20")
|
||||
check("tw: text", d["text"] == "hello")
|
||||
check("tw: author.screenName", d["author"]["screenName"] == "jack")
|
||||
check("tw: author.name", d["author"]["name"] == "Jack")
|
||||
check("tw: metrics.views", d["metrics"]["views"] == 99)
|
||||
check("tw: createdAtISO", d["createdAtISO"] == "2018-10-10T20:19:24+00:00")
|
||||
check("tw: isRetweet False", d["isRetweet"] is False)
|
||||
check("tw: photo media", d["media"][0] == {"type": "photo", "url": "https://pbs.twimg.com/media/AAA.jpg"})
|
||||
vid = d["media"][1]
|
||||
check("tw: video type", vid["type"] == "video")
|
||||
check("tw: video poster url", vid["url"] == "https://pbs.twimg.com/poster.jpg")
|
||||
check("tw: video variant url", vid["variants"][0]["url"] == "https://video.twimg.com/x.mp4")
|
||||
check("tw: video variant bitrate", vid["variants"][0]["bitrate"] == 832000)
|
||||
check("tw: video variant contentType", vid["variants"][0]["contentType"] == "video/mp4")
|
||||
|
||||
tw_rt = ns(id_str="21", rawContent="rt", user=user, retweetedTweet=ns(id="1"), media=None,
|
||||
likeCount=0, retweetCount=0, replyCount=0, viewCount=0, date=None)
|
||||
check("tw: isRetweet True", xa.twscrape_tweet_to_dict(tw_rt)["isRetweet"] is True)
|
||||
check("tw: no media -> empty", xa.twscrape_tweet_to_dict(tw_rt)["media"] == [])
|
||||
|
||||
# ── parse_graphql_tweet_result: legacy-user shape ──
|
||||
result_legacy = {
|
||||
"rest_id": "100",
|
||||
"core": {"user_results": {"result": {
|
||||
"rest_id": "12",
|
||||
"legacy": {"name": "Jack", "screen_name": "jack",
|
||||
"profile_image_url_https": "https://pbs.twimg.com/pp.jpg"},
|
||||
}}},
|
||||
"views": {"count": "1234"},
|
||||
"legacy": {
|
||||
"full_text": "from timeline",
|
||||
"created_at": "Wed Oct 10 20:19:24 +0000 2018",
|
||||
"favorite_count": 7, "retweet_count": 3, "reply_count": 2,
|
||||
"extended_entities": {"media": [
|
||||
{"type": "photo", "media_url_https": "https://pbs.twimg.com/media/P.jpg"},
|
||||
{"type": "video", "media_url_https": "https://pbs.twimg.com/poster.jpg",
|
||||
"video_info": {"variants": [
|
||||
{"url": "https://video.twimg.com/v.mp4", "bitrate": 256000, "content_type": "video/mp4"},
|
||||
{"url": "https://video.twimg.com/v.m3u8", "content_type": "application/x-mpegURL"},
|
||||
]}},
|
||||
]},
|
||||
},
|
||||
}
|
||||
g = xa.parse_graphql_tweet_result(result_legacy)
|
||||
check("gql: id", g["id"] == "100")
|
||||
check("gql: text", g["text"] == "from timeline")
|
||||
check("gql: screenName(legacy)", g["author"]["screenName"] == "jack")
|
||||
check("gql: name(legacy)", g["author"]["name"] == "Jack")
|
||||
check("gql: views parsed", g["metrics"]["views"] == 1234)
|
||||
check("gql: likes", g["metrics"]["likes"] == 7)
|
||||
check("gql: createdAtISO", g["createdAtISO"].startswith("2018-10-10T20:19:24"))
|
||||
check("gql: photo media", g["media"][0]["type"] == "photo")
|
||||
check("gql: video variant only-with-url", len(g["media"][1]["variants"]) == 2)
|
||||
check("gql: video contentType mapped", g["media"][1]["variants"][0]["contentType"] == "video/mp4")
|
||||
|
||||
# ── parse_graphql_tweet_result: core-user shape (X migration) ──
|
||||
result_core = {
|
||||
"rest_id": "101",
|
||||
"core": {"user_results": {"result": {
|
||||
"rest_id": "12",
|
||||
"core": {"name": "Jill", "screen_name": "jill"},
|
||||
"avatar": {"image_url": "https://pbs.twimg.com/jill.jpg"},
|
||||
}}},
|
||||
"views": {"count": "5"},
|
||||
"legacy": {"full_text": "core shape", "created_at": "", "favorite_count": 1,
|
||||
"retweet_count": 0, "reply_count": 0},
|
||||
}
|
||||
gc = xa.parse_graphql_tweet_result(result_core)
|
||||
check("gql-core: screenName", gc["author"]["screenName"] == "jill")
|
||||
check("gql-core: name", gc["author"]["name"] == "Jill")
|
||||
check("gql-core: avatar", gc["author"]["profileImageUrl"] == "https://pbs.twimg.com/jill.jpg")
|
||||
|
||||
# ── parse_graphql_tweet_result: visibility wrapper + no-legacy ──
|
||||
wrapped = {"__typename": "TweetWithVisibilityResults", "tweet": result_legacy}
|
||||
check("gql: visibility wrapper unwrapped", xa.parse_graphql_tweet_result(wrapped)["id"] == "100")
|
||||
check("gql: no legacy -> None", xa.parse_graphql_tweet_result({"rest_id": "9"}) is None)
|
||||
check("gql: non-dict -> None", xa.parse_graphql_tweet_result(None) is None)
|
||||
|
||||
# ── parse_home_timeline ──
|
||||
payload = {"data": {"home": {"home_timeline_urt": {"instructions": [
|
||||
{"type": "TimelineClearCache"},
|
||||
{"type": "TimelineAddEntries", "entries": [
|
||||
{"entryId": "tweet-100", "content": {"itemContent": {"tweet_results": {"result": result_legacy}}}},
|
||||
{"entryId": "cursor-top-x", "content": {}},
|
||||
{"entryId": "tweet-101", "content": {"itemContent": {"tweet_results": {"result": result_core}}}},
|
||||
]},
|
||||
]}}}}
|
||||
rows = xa.parse_home_timeline(payload, limit=50)
|
||||
check("home: 2 tweets extracted", len(rows) == 2)
|
||||
check("home: order preserved", rows[0]["id"] == "100" and rows[1]["id"] == "101")
|
||||
check("home: limit respected", len(xa.parse_home_timeline(payload, limit=1)) == 1)
|
||||
check("home: empty payload", xa.parse_home_timeline({}, limit=10) == [])
|
||||
|
||||
# module-nested tweets (TimelineModule items[] + TimelineAddToModule moduleItems[])
|
||||
payload_mod = {"data": {"home": {"home_timeline_urt": {"instructions": [
|
||||
{"type": "TimelineAddEntries", "entries": [
|
||||
{"entryId": "home-conversation-1", "content": {"items": [
|
||||
{"item": {"itemContent": {"tweet_results": {"result": result_legacy}}}},
|
||||
]}},
|
||||
]},
|
||||
{"type": "TimelineAddToModule", "moduleItems": [
|
||||
{"item": {"itemContent": {"tweet_results": {"result": result_core}}}},
|
||||
]},
|
||||
]}}}}
|
||||
mrows = xa.parse_home_timeline(payload_mod, limit=50)
|
||||
check("home: module items extracted", len(mrows) == 2 and mrows[0]["id"] == "100" and mrows[1]["id"] == "101")
|
||||
check("home: who-to-follow user skipped", xa.parse_home_timeline(
|
||||
{"data": {"home": {"home_timeline_urt": {"instructions": [
|
||||
{"type": "TimelineAddEntries", "entries": [
|
||||
{"entryId": "cursor-top", "content": {"cursorType": "Top"}},
|
||||
]}]}}}}, limit=10) == [])
|
||||
|
||||
# ── emit (optional, needs PyYAML) ──
|
||||
try:
|
||||
import io
|
||||
import yaml # noqa: F401
|
||||
buf = io.StringIO()
|
||||
old = sys.stdout
|
||||
sys.stdout = buf
|
||||
try:
|
||||
xa.emit([g])
|
||||
finally:
|
||||
sys.stdout = old
|
||||
parsed = yaml.safe_load(buf.getvalue())
|
||||
check("emit: ok flag", parsed["ok"] is True)
|
||||
check("emit: data roundtrip", parsed["data"][0]["id"] == "100")
|
||||
check("emit: unicode preserved", "from timeline" in buf.getvalue())
|
||||
except ImportError:
|
||||
print("SKIP emit tests (PyYAML not installed)")
|
||||
|
||||
print()
|
||||
if _fails:
|
||||
print(f"{len(_fails)} FAILED: {_fails}")
|
||||
sys.exit(1)
|
||||
print("ALL PASSED")
|
||||
@@ -0,0 +1,543 @@
|
||||
#!/usr/bin/env python3
|
||||
"""twitter-cli 互換アダプタ (twscrape + twikit バックエンド)
|
||||
|
||||
なぜ存在するか:
|
||||
X.com が 2026 年に Web クライアントを新バンドル (x-web / `sign.o*.js`) へ移行し、
|
||||
twitter-cli が依存する x_client_transaction (1.0.2) の `ondemand.s` 解析が壊れた。
|
||||
transaction-id を生成できず全 X ツールが HTTP 404 で全滅 (`exited with code 1`)。
|
||||
twscrape の `XClIdGen` は新バンドルに追従済みなので、txid 生成と GraphQL 取得を
|
||||
twscrape に任せ、twitter-cli が出していた YAML と同じ形を stdout に吐く。
|
||||
既存の src/engine/tools/x.ts は無改修で、config の tools.x_cli_command を
|
||||
このアダプタに向けるだけで動く。
|
||||
|
||||
サブコマンド (twitter-cli と同じ呼ばれ方):
|
||||
search <query> -t <tab> --max N --yaml [--full-text] [--compact]
|
||||
user-posts <username> --max N --yaml [...]
|
||||
tweet <id|url> --yaml [...]
|
||||
feed --max N --yaml [-t following] [...] # ホームタイムライン
|
||||
--version
|
||||
|
||||
認証 (env 経由、x.ts が設定):
|
||||
TWITTER_AUTH_TOKEN, TWITTER_CT0
|
||||
|
||||
出力 YAML (x.ts の parseXPostsFromYaml / downloadTweetMedia が解釈する形):
|
||||
ok: true
|
||||
schema_version: '1'
|
||||
data:
|
||||
- id, text, author{name,screenName,profileImageUrl,id},
|
||||
metrics{likes,retweets,replies,views}, createdAtISO, isRetweet,
|
||||
media[]{type, url, variants[]{url,bitrate,contentType}}
|
||||
|
||||
注意: twscrape / twikit / httpx は遅延 import (関数内)。pure なマッピング関数だけ
|
||||
なら依存なしで import でき、test_x_adapter.py がネットワーク・依存なしで回る。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from email.utils import parsedate_to_datetime
|
||||
|
||||
VERSION = "x-adapter 1.0.0 (twscrape backend)"
|
||||
|
||||
# twitter-cli の YAML スキーマバージョン (x.ts は使わないが互換のため踏襲)
|
||||
SCHEMA_VERSION = "1"
|
||||
|
||||
|
||||
# ── pure helpers (依存なし・テスト対象) ──────────────────────────────
|
||||
|
||||
def parse_tweet_ref(raw: str) -> str | None:
|
||||
"""tweet ID もしくは status URL から数値 ID を取り出す。"""
|
||||
s = (raw or "").strip()
|
||||
if not s:
|
||||
return None
|
||||
if s.isdigit():
|
||||
return s
|
||||
m = re.search(r"(?:x\.com|twitter\.com)/[^/]+/status(?:es)?/(\d+)", s)
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
def twitter_time_iso(created_at: str) -> str:
|
||||
"""GraphQL legacy の 'Wed Oct 10 20:19:24 +0000 2018' を ISO8601 へ。"""
|
||||
if not created_at:
|
||||
return ""
|
||||
try:
|
||||
return parsedate_to_datetime(created_at).isoformat()
|
||||
except Exception:
|
||||
return created_at
|
||||
|
||||
|
||||
def _iso_from_dt(dt) -> str:
|
||||
if dt is None:
|
||||
return ""
|
||||
try:
|
||||
return dt.isoformat()
|
||||
except Exception:
|
||||
return str(dt)
|
||||
|
||||
|
||||
def _as_dict(obj) -> dict:
|
||||
if isinstance(obj, dict):
|
||||
return obj
|
||||
return getattr(obj, "__dict__", {}) or {}
|
||||
|
||||
|
||||
def _dig(d, *keys, default=None):
|
||||
"""ネストした dict を安全にたどる。"""
|
||||
cur = d
|
||||
for k in keys:
|
||||
if not isinstance(cur, dict):
|
||||
return default
|
||||
cur = cur.get(k)
|
||||
if cur is None:
|
||||
return default
|
||||
return cur
|
||||
|
||||
|
||||
# ── twscrape Tweet オブジェクト → YAML dict ───────────────────────────
|
||||
|
||||
def _media_from_twscrape(t) -> list:
|
||||
out: list = []
|
||||
media = getattr(t, "media", None)
|
||||
if not media:
|
||||
return out
|
||||
for p in getattr(media, "photos", None) or []:
|
||||
url = getattr(p, "url", None) if not isinstance(p, dict) else p.get("url")
|
||||
if url:
|
||||
out.append({"type": "photo", "url": url})
|
||||
for v in getattr(media, "videos", None) or []:
|
||||
vd = _as_dict(v)
|
||||
variants = []
|
||||
for var in (getattr(v, "variants", None) or vd.get("variants") or []):
|
||||
d = _as_dict(var)
|
||||
vu = d.get("url")
|
||||
if vu:
|
||||
variants.append({
|
||||
"url": vu,
|
||||
"bitrate": d.get("bitrate") or 0,
|
||||
"contentType": d.get("contentType") or d.get("content_type") or "",
|
||||
})
|
||||
out.append({
|
||||
"type": "video",
|
||||
"url": (getattr(v, "thumbnailUrl", None) or vd.get("thumbnailUrl") or ""),
|
||||
"variants": variants,
|
||||
})
|
||||
for g in getattr(media, "animated", None) or []:
|
||||
gd = _as_dict(g)
|
||||
variants = []
|
||||
for var in (getattr(g, "variants", None) or gd.get("variants") or []):
|
||||
d = _as_dict(var)
|
||||
if d.get("url"):
|
||||
variants.append({"url": d["url"], "bitrate": d.get("bitrate") or 0,
|
||||
"contentType": d.get("contentType") or d.get("content_type") or ""})
|
||||
out.append({
|
||||
"type": "animated_gif",
|
||||
"url": (getattr(g, "thumbnailUrl", None) or gd.get("thumbnailUrl") or ""),
|
||||
"variants": variants,
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def twscrape_tweet_to_dict(t) -> dict:
|
||||
u = getattr(t, "user", None)
|
||||
return {
|
||||
"id": str(getattr(t, "id_str", None) or getattr(t, "id", "") or ""),
|
||||
"text": getattr(t, "rawContent", "") or "",
|
||||
"author": {
|
||||
"id": str(getattr(u, "id_str", "") or "") if u else "",
|
||||
"name": (getattr(u, "displayname", "") or "") if u else "",
|
||||
"screenName": (getattr(u, "username", "") or "") if u else "",
|
||||
"profileImageUrl": (getattr(u, "profileImageUrl", "") or "") if u else "",
|
||||
},
|
||||
"metrics": {
|
||||
"likes": getattr(t, "likeCount", 0) or 0,
|
||||
"retweets": getattr(t, "retweetCount", 0) or 0,
|
||||
"replies": getattr(t, "replyCount", 0) or 0,
|
||||
"views": getattr(t, "viewCount", 0) or 0,
|
||||
},
|
||||
"createdAtISO": _iso_from_dt(getattr(t, "date", None)),
|
||||
"isRetweet": getattr(t, "retweetedTweet", None) is not None,
|
||||
"media": _media_from_twscrape(t),
|
||||
}
|
||||
|
||||
|
||||
# ── raw GraphQL (home timeline) result → YAML dict ────────────────────
|
||||
|
||||
def parse_graphql_tweet_result(result: dict) -> dict | None:
|
||||
"""GraphQL の tweet_results.result を YAML dict に変換。
|
||||
|
||||
X は user フィールドを legacy → core へ移行中なので両方から拾う。
|
||||
"""
|
||||
if not isinstance(result, dict):
|
||||
return None
|
||||
if result.get("__typename") == "TweetWithVisibilityResults":
|
||||
result = result.get("tweet", result)
|
||||
legacy = result.get("legacy") or {}
|
||||
if not legacy:
|
||||
return None
|
||||
|
||||
user_result = _dig(result, "core", "user_results", "result", default={}) or {}
|
||||
user_legacy = user_result.get("legacy") or {}
|
||||
user_core = user_result.get("core") or {}
|
||||
views = _dig(result, "views", "count", default=0)
|
||||
|
||||
media = []
|
||||
for me in (_dig(legacy, "extended_entities", "media", default=[]) or []):
|
||||
mt = me.get("type")
|
||||
if mt == "photo":
|
||||
media.append({"type": "photo", "url": me.get("media_url_https")})
|
||||
elif mt in ("video", "animated_gif"):
|
||||
variants = [
|
||||
{"url": v.get("url"), "bitrate": v.get("bitrate", 0),
|
||||
"contentType": v.get("content_type", "")}
|
||||
for v in (_dig(me, "video_info", "variants", default=[]) or [])
|
||||
if v.get("url")
|
||||
]
|
||||
media.append({"type": mt, "url": me.get("media_url_https"), "variants": variants})
|
||||
|
||||
return {
|
||||
"id": str(result.get("rest_id") or legacy.get("id_str") or ""),
|
||||
"text": legacy.get("full_text", "") or "",
|
||||
"author": {
|
||||
"id": str(user_result.get("rest_id", "") or ""),
|
||||
"name": user_legacy.get("name") or user_core.get("name") or "",
|
||||
"screenName": (user_legacy.get("screen_name")
|
||||
or user_core.get("screen_name") or ""),
|
||||
"profileImageUrl": (user_legacy.get("profile_image_url_https")
|
||||
or _dig(user_result, "avatar", "image_url", default="") or ""),
|
||||
},
|
||||
"metrics": {
|
||||
"likes": legacy.get("favorite_count", 0) or 0,
|
||||
"retweets": legacy.get("retweet_count", 0) or 0,
|
||||
"replies": legacy.get("reply_count", 0) or 0,
|
||||
"views": int(views) if str(views).isdigit() else 0,
|
||||
},
|
||||
"createdAtISO": twitter_time_iso(legacy.get("created_at", "")),
|
||||
"isRetweet": "retweeted_status_result" in legacy,
|
||||
"media": media,
|
||||
}
|
||||
|
||||
|
||||
def _collect_tweet_results(node, out: list, limit: int) -> None:
|
||||
"""entry / module item から tweet_results.result を拾って out に積む。
|
||||
|
||||
通常ツイート (content.itemContent...) と、モジュール内アイテム
|
||||
(content.items[].item.itemContent... / moduleItems[]) の両方をたどる。
|
||||
ツイート以外 (who-to-follow ユーザー, cursor) は parse_graphql_tweet_result が
|
||||
None を返すので自然に除外される。
|
||||
"""
|
||||
if len(out) >= limit:
|
||||
return
|
||||
content = node.get("content") or node.get("item") or {}
|
||||
res = _dig(content, "itemContent", "tweet_results", "result")
|
||||
if res:
|
||||
row = parse_graphql_tweet_result(res)
|
||||
if row:
|
||||
out.append(row)
|
||||
return
|
||||
for it in (content.get("items") or []):
|
||||
_collect_tweet_results(it, out, limit)
|
||||
if len(out) >= limit:
|
||||
return
|
||||
|
||||
|
||||
def parse_home_timeline(payload: dict, limit: int) -> list:
|
||||
"""HomeTimeline / HomeLatestTimeline の GraphQL レスポンスを dict 配列へ。
|
||||
|
||||
TimelineAddEntries の entries[] と TimelineAddToModule の moduleItems[] の
|
||||
両方を走査する。
|
||||
"""
|
||||
out: list = []
|
||||
instructions = (_dig(payload, "data", "home", "home_timeline_urt", "instructions", default=[]) or [])
|
||||
for ins in instructions:
|
||||
itype = ins.get("type")
|
||||
if itype == "TimelineAddEntries":
|
||||
for entry in ins.get("entries", []) or []:
|
||||
_collect_tweet_results(entry, out, limit)
|
||||
if len(out) >= limit:
|
||||
return out
|
||||
elif itype == "TimelineAddToModule":
|
||||
for it in ins.get("moduleItems", []) or []:
|
||||
_collect_tweet_results(it, out, limit)
|
||||
if len(out) >= limit:
|
||||
return out
|
||||
return out
|
||||
|
||||
|
||||
# ── 出力 ──────────────────────────────────────────────────────────────
|
||||
|
||||
def emit(rows: list) -> None:
|
||||
import yaml # PyYAML
|
||||
doc = {"ok": True, "schema_version": SCHEMA_VERSION, "data": rows}
|
||||
sys.stdout.write(yaml.safe_dump(doc, allow_unicode=True, sort_keys=False))
|
||||
|
||||
|
||||
def fail(message: str, code: int = 1) -> None:
|
||||
"""twitter-cli と同じく非ゼロ終了 + stderr。x.ts がエラー扱いにする。"""
|
||||
sys.stderr.write(message.rstrip() + "\n")
|
||||
sys.exit(code)
|
||||
|
||||
|
||||
def _silence_logs() -> None:
|
||||
"""twscrape は loguru で出力する。万一 stdout に乗ると x.ts の YAML.parse が
|
||||
壊れるので、全 loguru sink を落とす (現状は stderr 行きで実害なしだが防御的に)。"""
|
||||
try:
|
||||
from loguru import logger as _loguru
|
||||
_loguru.remove()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# ── backend (twscrape / twikit、遅延 import) ──────────────────────────
|
||||
|
||||
def _creds() -> tuple[str, str]:
|
||||
import os
|
||||
auth = os.environ.get("TWITTER_AUTH_TOKEN", "").strip()
|
||||
ct0 = os.environ.get("TWITTER_CT0", "").strip()
|
||||
if not auth or not ct0:
|
||||
fail("x-adapter: TWITTER_AUTH_TOKEN / TWITTER_CT0 are not set. "
|
||||
"Configure tools.x_auth_token / tools.x_ct0 in config.yaml.")
|
||||
return auth, ct0
|
||||
|
||||
|
||||
_TMP_PREFIX = "xadapter-"
|
||||
|
||||
|
||||
def _cleanup_stale_tmp() -> None:
|
||||
"""過去の SIGKILL タイムアウト等で残った temp dir (cookie 入り sqlite) を掃除。
|
||||
|
||||
x.ts はタイムアウト時に SIGKILL するため finally が走らず temp dir が残りうる。
|
||||
起動時に 1 時間以上前の `xadapter-*` を削除し、平文 cookie の /tmp 残留を抑える。
|
||||
"""
|
||||
import glob
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import time
|
||||
try:
|
||||
now = time.time()
|
||||
for d in glob.glob(os.path.join(tempfile.gettempdir(), _TMP_PREFIX + "*")):
|
||||
try:
|
||||
if os.path.isdir(d) and now - os.path.getmtime(d) > 3600:
|
||||
shutil.rmtree(d, ignore_errors=True)
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def _ensure_alive(api) -> None:
|
||||
"""直近のリクエストでアカウントが 401/banned 扱いになっていないか確認。
|
||||
|
||||
twscrape は 401/429 を例外にせずログして停止し、空イテレーションを返す。空結果が
|
||||
『本当に0件』か『認証切れ/レート制限』かを区別するため、停止時に無効化される
|
||||
account.active を見て、無効なら nonzero で fail する (x.ts がエラー表示)。
|
||||
"""
|
||||
try:
|
||||
accs = await api.pool.get_all()
|
||||
except Exception:
|
||||
return
|
||||
if accs and not getattr(accs[0], "active", True):
|
||||
fail("x-adapter: X rejected the session (cookies expired/banned or rate-limited). "
|
||||
"Refresh tools.x_auth_token / tools.x_ct0, or wait out the rate limit.")
|
||||
|
||||
|
||||
async def _make_api():
|
||||
"""cookie だけで twscrape API を用意 (login_all は呼ばない)。"""
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
from twscrape import API
|
||||
_silence_logs() # twscrape import 後に呼ぶ (import 時に loguru sink が張られるため)
|
||||
auth, ct0 = _creds()
|
||||
tmpdir = tempfile.mkdtemp(prefix=_TMP_PREFIX)
|
||||
db = os.path.join(tmpdir, "accounts.db")
|
||||
api = API(db)
|
||||
await api.pool.add_account(
|
||||
"x_adapter", "-", "x_adapter@local", "-",
|
||||
cookies=f"auth_token={auth}; ct0={ct0}",
|
||||
)
|
||||
return api, tmpdir, shutil
|
||||
|
||||
|
||||
def _search_product(tab: str) -> str:
|
||||
t = (tab or "Latest").strip().lower()
|
||||
return {
|
||||
"latest": "Latest", "top": "Top",
|
||||
"photos": "Media", "videos": "Media", "media": "Media",
|
||||
}.get(t, "Latest")
|
||||
|
||||
|
||||
async def cmd_search(args) -> None:
|
||||
api, tmpdir, shutil = await _make_api()
|
||||
try:
|
||||
rows = []
|
||||
kv = {"product": _search_product(args.tab)}
|
||||
async for t in api.search(args.query, limit=args.max, kv=kv):
|
||||
rows.append(twscrape_tweet_to_dict(t))
|
||||
if len(rows) >= args.max:
|
||||
break
|
||||
if not rows:
|
||||
await _ensure_alive(api) # 空が認証切れ由来なら nonzero fail
|
||||
emit(rows)
|
||||
finally:
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
|
||||
|
||||
async def cmd_user_posts(args) -> None:
|
||||
api, tmpdir, shutil = await _make_api()
|
||||
try:
|
||||
login = args.username.lstrip("@").strip()
|
||||
user = await api.user_by_login(login)
|
||||
if not user:
|
||||
await _ensure_alive(api) # 認証切れなら not found より先に auth エラー
|
||||
fail(f"x-adapter: user '{login}' not found")
|
||||
rows = []
|
||||
async for t in api.user_tweets(user.id, limit=args.max):
|
||||
rows.append(twscrape_tweet_to_dict(t))
|
||||
if len(rows) >= args.max:
|
||||
break
|
||||
if not rows:
|
||||
await _ensure_alive(api)
|
||||
emit(rows)
|
||||
finally:
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
|
||||
|
||||
async def cmd_tweet(args) -> None:
|
||||
tid = parse_tweet_ref(args.tweet)
|
||||
if not tid:
|
||||
fail(f"x-adapter: could not parse tweet id from '{args.tweet}'")
|
||||
api, tmpdir, shutil = await _make_api()
|
||||
try:
|
||||
t = await api.tweet_details(int(tid))
|
||||
if not t:
|
||||
await _ensure_alive(api) # 認証切れなら nonzero
|
||||
fail(f"x-adapter: tweet {tid} not found or inaccessible")
|
||||
emit([twscrape_tweet_to_dict(t)])
|
||||
finally:
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
|
||||
|
||||
async def cmd_feed(args) -> None:
|
||||
"""ホームタイムライン。twscrape の txid + twikit の endpoint で raw GraphQL。"""
|
||||
import httpx
|
||||
from urllib.parse import urlparse
|
||||
import twscrape.xclid as xc
|
||||
from twikit.client.gql import Endpoint
|
||||
from twikit.constants import FEATURES
|
||||
_silence_logs() # twscrape import 後に呼ぶ
|
||||
|
||||
auth, ct0 = _creds()
|
||||
following = (args.tab or "").strip().lower() == "following"
|
||||
url = Endpoint.HOME_LATEST_TIMELINE if following else Endpoint.HOME_TIMELINE
|
||||
path = urlparse(url).path
|
||||
query_id = path.rstrip("/").split("/")[-2]
|
||||
# X web app が使う公開 web bearer (guest/web 共通の定数)
|
||||
bearer = ("AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs"
|
||||
"%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA")
|
||||
|
||||
gen = await xc.XClIdGen.create()
|
||||
tid = gen.calc("POST", path)
|
||||
variables = {
|
||||
"count": args.max,
|
||||
"includePromotedContent": False,
|
||||
"latestControlAvailable": True,
|
||||
"requestContext": "launch",
|
||||
"seenTweetIds": [],
|
||||
}
|
||||
body = {"variables": variables, "features": FEATURES, "queryId": query_id}
|
||||
headers = {
|
||||
"authorization": "Bearer " + bearer,
|
||||
"x-csrf-token": ct0,
|
||||
"x-twitter-auth-type": "OAuth2Session",
|
||||
"x-twitter-active-user": "yes",
|
||||
"content-type": "application/json",
|
||||
"x-client-transaction-id": tid,
|
||||
"cookie": f"auth_token={auth}; ct0={ct0}",
|
||||
"user-agent": ("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"),
|
||||
}
|
||||
async with httpx.AsyncClient(timeout=30) as h:
|
||||
r = await h.post(url, json=body, headers=headers)
|
||||
if r.status_code != 200:
|
||||
# レスポンス本文はそのまま吐かない (デバッグ用に X の error code/message のみ抽出)
|
||||
fail(f"x-adapter: home timeline HTTP {r.status_code}{_x_error_suffix(r)}")
|
||||
try:
|
||||
payload = r.json()
|
||||
except Exception:
|
||||
fail("x-adapter: home timeline returned non-JSON")
|
||||
# 200 でも GraphQL の errors が返ることがある (認証劣化など)。entries が無く errors が
|
||||
# あるなら成功偽装せず fail。
|
||||
rows = parse_home_timeline(payload, args.max)
|
||||
if not rows and isinstance(payload, dict) and payload.get("errors"):
|
||||
msgs = "; ".join(str(e.get("message", "")) for e in payload["errors"] if isinstance(e, dict))
|
||||
fail(f"x-adapter: home timeline error: {msgs[:200] or 'unknown'}")
|
||||
emit(rows)
|
||||
|
||||
|
||||
def _x_error_suffix(resp) -> str:
|
||||
"""X のエラーレスポンスから errors[].message だけ安全に取り出す (cookie/header は出さない)。"""
|
||||
try:
|
||||
data = resp.json()
|
||||
errs = data.get("errors") if isinstance(data, dict) else None
|
||||
if errs:
|
||||
return ": " + "; ".join(str(e.get("message", "")) for e in errs if isinstance(e, dict))[:200]
|
||||
except Exception:
|
||||
pass
|
||||
return ""
|
||||
|
||||
|
||||
def _clamp_max(v) -> int:
|
||||
try:
|
||||
n = int(v)
|
||||
except Exception:
|
||||
n = 20
|
||||
return max(1, min(n, 50))
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(prog="x-adapter", add_help=True)
|
||||
p.add_argument("--version", action="store_true")
|
||||
sub = p.add_subparsers(dest="command")
|
||||
|
||||
def common(sp):
|
||||
sp.add_argument("--max", type=_clamp_max, default=20)
|
||||
sp.add_argument("--yaml", action="store_true")
|
||||
sp.add_argument("--full-text", action="store_true")
|
||||
sp.add_argument("--compact", action="store_true")
|
||||
|
||||
sp = sub.add_parser("search"); sp.add_argument("query"); sp.add_argument("-t", "--tab", default="Latest"); common(sp)
|
||||
sp = sub.add_parser("user-posts"); sp.add_argument("username"); common(sp)
|
||||
sp = sub.add_parser("tweet"); sp.add_argument("tweet"); common(sp)
|
||||
sp = sub.add_parser("feed"); sp.add_argument("-t", "--tab", default="for_you"); common(sp)
|
||||
return p
|
||||
|
||||
|
||||
def main(argv=None) -> None:
|
||||
import asyncio
|
||||
args = build_parser().parse_args(argv)
|
||||
if args.version:
|
||||
print(VERSION)
|
||||
return
|
||||
handlers = {
|
||||
"search": cmd_search,
|
||||
"user-posts": cmd_user_posts,
|
||||
"tweet": cmd_tweet,
|
||||
"feed": cmd_feed,
|
||||
}
|
||||
handler = handlers.get(args.command)
|
||||
if not handler:
|
||||
fail("x-adapter: no subcommand. Use search|user-posts|tweet|feed.")
|
||||
_cleanup_stale_tmp() # 過去の SIGKILL タイムアウトで残った cookie 入り temp を掃除
|
||||
try:
|
||||
asyncio.run(handler(args))
|
||||
except SystemExit:
|
||||
raise
|
||||
except Exception as e:
|
||||
fail(f"x-adapter: {type(e).__name__}: {e}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user