sync: update from private repo (0cddeaa)

src/engine/tools/web.ts

@@ -8,6 +8,12 @@ import { htmlToText } from './shared/html.js';
 import * as fs from 'fs';
 import * as path from 'path';
 import * as crypto from 'crypto';
+import {
+  looksLikeBinaryBytes,
+  decodeText,
+  SNIFF_HEAD_BYTES,
+  type BinaryVerdict,
+} from './binary-detect.js';
 
 const BINARY_CONTENT_TYPE_PREFIXES = [
   'application/pdf',
@@ -20,6 +26,9 @@ const BINARY_CONTENT_TYPE_PREFIXES = [
   'video/',
 ];
 
+// WebFetch text body は最大 5MB で打ち切る（巨大 HTML による context 膨張防止）
+const MAX_WEBFETCH_BODY_BYTES = 5 * 1024 * 1024;
+
 // --- ツール定義 ---
 
 const WEBSEARCH_DEF: ToolDef = {
@@ -737,6 +746,66 @@ function formatResults(results: SearchResult[]): string {
 
 // --- WebFetch 実装 ---
 
+export type SniffResult =
+  | { binary: true; reason: string }
+  | { binary: false; text: string; truncated: boolean };
+
+/**
+ * Stream a fetch Response body, sniff the first SNIFF_HEAD_BYTES for binary
+ * content, and either block (binary) or strict-decode the (capped) text body.
+ * Never uses response.text() — that silently produces U+FFFD from binary.
+ */
+export async function sniffAndDecodeBody(
+  response: { body: ReadableStream<Uint8Array> | null },
+): Promise<SniffResult> {
+  const reader = response.body?.getReader();
+  if (!reader) return { binary: false, text: '', truncated: false };
+
+  const chunks: Buffer[] = [];
+  let total = 0;
+  let verdict: BinaryVerdict | null = null;
+  let truncated = false;
+
+  try {
+    for (;;) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      if (!value || value.byteLength === 0) continue;
+      chunks.push(Buffer.from(value));
+      total += value.byteLength;
+
+      if (!verdict && total >= SNIFF_HEAD_BYTES) {
+        const head = Buffer.concat(chunks).subarray(0, SNIFF_HEAD_BYTES);
+        verdict = looksLikeBinaryBytes(head);
+        if (verdict.binary) {
+          await reader.cancel();
+          return { binary: true, reason: verdict.reason };
+        }
+      }
+
+      if (total >= MAX_WEBFETCH_BODY_BYTES) {
+        truncated = true;
+        await reader.cancel();
+        break;
+      }
+    }
+  } finally {
+    try { reader.releaseLock(); } catch { /* cancel() may or may not have released the lock depending on runtime */ }
+  }
+
+  let full = Buffer.concat(chunks);
+  if (truncated) full = full.subarray(0, MAX_WEBFETCH_BODY_BYTES);
+
+  if (!verdict) {
+    verdict = looksLikeBinaryBytes(full.subarray(0, SNIFF_HEAD_BYTES));
+    if (verdict.binary) return { binary: true, reason: verdict.reason };
+  }
+
+  const text = decodeText(full, verdict.encoding);
+  if (text === null) return { binary: true, reason: 'utf8-decode-fail' };
+  return { binary: false, text, truncated };
+}
+
 async function executeWebFetch(
   input: Record<string, unknown>,
   ctx: ToolContext,
@@ -848,8 +917,25 @@ async function executeWebFetch(
       };
     }
 
-    const html = await response.text();
-    const text = htmlToText(html);
+    const sniffed = await sniffAndDecodeBody(response);
+    if (sniffed.binary) {
+      appendWebFetchHistory(ctx, {
+        timestamp: new Date().toISOString(),
+        url: rawUrl,
+        selector,
+        status: response.status,
+        contentType,
+        outcome: 'binary_blocked',
+        error: `binary content detected (${sniffed.reason})`,
+      });
+      return {
+        output: `WebFetch blocked binary content from "${rawUrl}" (detected: ${sniffed.reason}). コンテキストに展開していません。DownloadFile で input/ に保存し、ReadExcel/ReadPdf 等で処理してください。`,
+        isError: true,
+      };
+    }
+    const text =
+      htmlToText(sniffed.text) +
+      (sniffed.truncated ? '\n\n[truncated: body exceeded 5MB]' : '');
 
     // vlmEnabled 時はファーストビューのスクショを並行取得して画像を添付する。
     // 失敗時は警告ログのみで WebFetch 自体は成功扱いとする。