import * as fs from 'fs'; import * as path from 'path'; import sharp from 'sharp'; import { ToolDef } from '../../llm/openai-compat.js'; import type { ToolContext, ToolResult } from './core.js'; import { resolveAndGuard, resolveOutputPathWithin } from './core.js'; import { logger } from '../../logger.js'; import { recordLlmUsage } from '../../llm/usage-recorder.js'; // --- Supported image extensions --- const SUPPORTED_EXTENSIONS = new Set(['png', 'jpg', 'jpeg', 'gif', 'webp', 'bmp']); const DEFAULT_IMAGE_PROMPT = '画像の内容を日本語で説明してください。'; // Normalize extension for MIME type (jpg -> jpeg) function extToMime(ext: string): string { if (ext === 'jpg') return 'jpeg'; return ext; } // --- SVG annotation helpers --- interface Annotation { type: 'rectangle' | 'arrow' | 'text'; x?: number; y?: number; width?: number; height?: number; from_x?: number; from_y?: number; to_x?: number; to_y?: number; text?: string; color?: string; label?: string; font_size?: number; } function computeScaling(imageWidth: number, imageHeight: number): { strokeWidth: number; fontSize: number } { const shortSide = Math.min(imageWidth, imageHeight); return { strokeWidth: Math.max(2, Math.min(5, Math.round(shortSide / 400))), fontSize: Math.max(12, Math.min(32, Math.round(shortSide / 40))), }; } function escapeXml(str: string): string { return str .replace(/&/g, '&') .replace(//g, '>') .replace(/"/g, '"') .replace(/'/g, '''); } // Convert a CSS color value into a string safe for use as an SVG `id`. // SVG ids referenced via `url(#...)` may not contain spaces, parens, or commas, // so functional notations like `rgb(255, 0, 0)` cannot be used as-is. function colorToMarkerId(color: string): string { return `ah-${color.replace(/[^a-zA-Z0-9]/g, '_')}`; } function buildAnnotationSvg( imageWidth: number, imageHeight: number, annotations: Annotation[], ): string { const { strokeWidth, fontSize: defaultFontSize } = computeScaling(imageWidth, imageHeight); const markerSize = strokeWidth * 3; const parts: string[] = []; parts.push(``); parts.push(''); // One arrowhead marker per unique color const arrowColors = new Set(); for (const a of annotations) { if (a.type === 'arrow') arrowColors.add(a.color ?? '#FF0000'); } for (const color of arrowColors) { const id = colorToMarkerId(color); parts.push( `` + `` + '', ); } parts.push(''); for (const a of annotations) { const color = a.color ?? '#FF0000'; const fontSize = a.font_size ?? defaultFontSize; const labelPadding = Math.round(fontSize * 0.3); switch (a.type) { case 'rectangle': { const x = a.x ?? 0; const y = a.y ?? 0; const w = a.width ?? 50; const h = a.height ?? 50; parts.push( ``, ); if (a.label) { const labelWidth = a.label.length * fontSize * 0.6 + labelPadding * 2; const labelHeight = fontSize + labelPadding * 2; const labelY = y - labelHeight; const textY = y - labelPadding; parts.push( ``, ); parts.push( `${escapeXml(a.label)}`, ); } break; } case 'arrow': { const fx = a.from_x ?? 0; const fy = a.from_y ?? 0; const tx = a.to_x ?? 0; const ty = a.to_y ?? 0; const markerId = colorToMarkerId(color); parts.push( ``, ); if (a.label) { const bgWidth = a.label.length * fontSize * 0.6 + labelPadding * 2; const bgHeight = fontSize + labelPadding * 2; parts.push( ``, ); parts.push( `${escapeXml(a.label)}`, ); } break; } case 'text': { const x = a.x ?? 0; const y = a.y ?? 0; const text = a.text ?? ''; if (text) { const bgWidth = text.length * fontSize * 0.6 + labelPadding * 2; const bgHeight = fontSize + labelPadding * 2; parts.push( ``, ); parts.push( `${escapeXml(text)}`, ); } break; } } } parts.push(''); return parts.join('\n'); } // --- Tool definition --- const READIMAGE_DEF: ToolDef = { type: 'function', function: { name: 'ReadImage', description: '画像ファイルを LLM に直接渡して内容を認識させる(VLM 対応 worker のみ)。詳細は ReadToolDoc({ name: "ReadImage" })。', parameters: { type: 'object', properties: { file_path: { type: 'string', description: 'workspace 内の画像ファイルパス' }, prompt: { type: 'string', description: '画像について質問したいこと(省略可)' }, }, required: ['file_path'], }, }, }; const ANNOTATEIMAGE_DEF: ToolDef = { type: 'function', function: { name: 'AnnotateImage', description: '画像上に矩形枠・矢印・テキストラベルを描画して output/ に保存する(元画像は変更されない)。詳細は ReadToolDoc({ name: "AnnotateImage" })。', parameters: { type: 'object', properties: { input_path: { type: 'string', description: 'workspace 内の元画像ファイルパス' }, output_path: { type: 'string', description: '出力先パス(output/ 配下)' }, annotations: { type: 'array', description: '描画する注釈の配列', items: { type: 'object', properties: { type: { type: 'string', enum: ['rectangle', 'arrow', 'text'], description: '注釈の種類' }, x: { type: 'number', description: 'rectangle/text: 左上 X 座標' }, y: { type: 'number', description: 'rectangle/text: 左上 Y 座標' }, width: { type: 'number', description: 'rectangle: 幅' }, height: { type: 'number', description: 'rectangle: 高さ' }, from_x: { type: 'number', description: 'arrow: 始点 X' }, from_y: { type: 'number', description: 'arrow: 始点 Y' }, to_x: { type: 'number', description: 'arrow: 終点 X' }, to_y: { type: 'number', description: 'arrow: 終点 Y' }, text: { type: 'string', description: 'text: 表示するテキスト' }, color: { type: 'string', description: '色(デフォルト: #FF0000)' }, label: { type: 'string', description: 'rectangle/arrow: ラベルテキスト' }, font_size: { type: 'number', description: 'フォントサイズ(省略時は画像サイズから自動算出)' }, }, required: ['type'], }, }, }, required: ['input_path', 'output_path', 'annotations'], }, }, }; export const TOOL_DEFS: Record = { ReadImage: READIMAGE_DEF, AnnotateImage: ANNOTATEIMAGE_DEF, }; // --- Tool execution --- export function resolveImagePath( filePath: string, ctx: ToolContext, ): ToolResult | { resolved: string; ext: string; dataUrl: string } { let resolved: string; try { resolved = resolveAndGuard(ctx.workspacePath, filePath); } catch (e) { return { output: (e as Error).message, isError: true }; } const ext = path.extname(resolved).replace('.', '').toLowerCase(); if (!SUPPORTED_EXTENSIONS.has(ext)) { return { output: `Unsupported image extension: "${ext}". Supported: ${[...SUPPORTED_EXTENSIONS].join(', ')}`, isError: true, }; } let base64data: string; try { const buf = fs.readFileSync(resolved); base64data = buf.toString('base64'); } catch (e) { return { output: `Failed to read image: ${(e as Error).message}`, isError: true }; } const mimeExt = extToMime(ext); return { resolved, ext, dataUrl: `data:image/${mimeExt};base64,${base64data}`, }; } export async function callVisionModel( dataUrl: string, prompt: string, ctx: ToolContext, ): Promise { const finalPrompt = prompt.trim() || DEFAULT_IMAGE_PROMPT; const toolsConfig = ctx.toolsConfig ?? {}; const visionModel = toolsConfig.visionModel ?? 'qwen2-vl:8b-instruct'; // Neutral placeholder — set tools.vision_base_url in config.yaml for a real endpoint. // (Workers usually override this with their own endpoint; see worker.ts) const visionBaseUrl = toolsConfig.visionBaseUrl ?? 'http://10.0.0.10:11434/v1'; const visionTimeout = (toolsConfig.visionTimeout ?? 60) * 1000; const visionMaxTokens = toolsConfig.visionMaxTokens ?? 1024; const requestBody = { model: visionModel, messages: [ { role: 'user', content: [ { type: 'text', text: finalPrompt }, { type: 'image_url', image_url: { url: dataUrl } }, ], }, ], max_tokens: visionMaxTokens, }; logger.debug(`[ReadImage] calling ${visionBaseUrl}/chat/completions model=${visionModel}`); const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), visionTimeout); try { const response = await fetch(`${visionBaseUrl}/chat/completions`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(requestBody), signal: controller.signal, }); if (!response.ok) { const text = await response.text().catch(() => ''); return { output: `Vision API error ${response.status}: ${text}`, isError: true }; } const json = (await response.json()) as { model?: string; choices?: Array<{ message?: { content?: string } }>; usage?: { prompt_tokens?: number; completion_tokens?: number }; }; const content = json.choices?.[0]?.message?.content; if (typeof content !== 'string') { return { output: 'Vision API returned no content', isError: true }; } // This vision call is a raw fetch outside OpenAICompatClient, so it would // otherwise miss the per-user usage ledger. Record it directly (issue // #499). Vision uses its own endpoint directly (never the gateway). let route = 'unknown'; try { route = new URL(visionBaseUrl).host || 'unknown'; } catch { /* keep 'unknown' */ } recordLlmUsage({ userId: ctx.userId ?? 'local', source: 'direct', model: json.model || visionModel, route, tokensIn: json.usage?.prompt_tokens ?? 0, tokensOut: json.usage?.completion_tokens ?? 0, }); return { output: content, isError: false }; } catch (e) { if ((e as Error).name === 'AbortError') { return { output: `Vision API timed out after ${visionTimeout / 1000}s`, isError: true }; } return { output: `Vision API request failed: ${(e as Error).message}`, isError: true }; } finally { clearTimeout(timer); } } async function executeReadImage( input: Record, ctx: ToolContext, ): Promise { if (!ctx.vlmEnabled) { return { output: 'ReadImage requires VLM-enabled worker (vlm: true)', isError: true }; } const filePath = input['file_path'] as string; const prompt = typeof input['prompt'] === 'string' ? input['prompt'] : ''; const resolvedImage = resolveImagePath(filePath, ctx); if ('isError' in resolvedImage) return resolvedImage; const description = prompt ? `画像を読み込みました: ${filePath}\n指示: ${prompt}` : `画像を読み込みました: ${filePath}`; return { output: description, isError: false, images: [{ dataUrl: resolvedImage.dataUrl, label: filePath }], }; } function isValidCssColor(color: string): boolean { if (typeof color !== 'string') return false; // Hex: #RGB, #RRGGBB, #RRGGBBAA if (/^#[0-9A-Fa-f]{3}$|^#[0-9A-Fa-f]{6}$|^#[0-9A-Fa-f]{8}$/.test(color)) return true; // Named colors (lowercase letters only — covers "red", "blue", "transparent", etc.) if (/^[a-z]+$/.test(color)) return true; // rgb(), rgba(), hsl(), hsla() — basic shape only if (/^(?:rgb|rgba|hsl|hsla)\(\s*[\d.,%\s/]+\s*\)$/.test(color)) return true; return false; } const VALID_TYPES = new Set(['rectangle', 'arrow', 'text']); async function executeAnnotateImage( input: Record, ctx: ToolContext, ): Promise { if (!ctx.editAllowed) { return { output: 'AnnotateImage is not allowed: edit flag is false', isError: true }; } const inputPath = input['input_path'] as string; const outputPath = input['output_path'] as string; const annotations = input['annotations'] as Annotation[]; if (!Array.isArray(annotations) || annotations.length === 0) { return { output: 'AnnotateImage requires at least one annotation', isError: true }; } for (let i = 0; i < annotations.length; i++) { const a = annotations[i]; if (!VALID_TYPES.has(a.type)) { return { output: `AnnotateImage failed: annotation[${i}].type "${a.type}" is invalid. Must be one of: rectangle, arrow, text.`, isError: true, }; } if (a.color !== undefined && !isValidCssColor(a.color)) { return { output: `AnnotateImage failed: annotation[${i}] has invalid color "${a.color}". Use formats like "#FF0000", "red", or "rgb(255,0,0)".`, isError: true, }; } } // Resolve and guard input path let resolvedInput: string; try { resolvedInput = resolveAndGuard(ctx.workspacePath, inputPath); } catch (e) { return { output: (e as Error).message, isError: true }; } // Resolve and guard output path (must be within output/) let resolvedOutput: string; try { resolvedOutput = resolveOutputPathWithin(ctx.workspacePath, outputPath, ['output']); } catch (e) { return { output: (e as Error).message, isError: true }; } try { // Read image metadata const metadata = await sharp(resolvedInput).metadata(); if (!metadata.width || !metadata.height) { return { output: `AnnotateImage failed: cannot determine image dimensions for ${inputPath}`, isError: true, }; } const imageWidth = metadata.width; const imageHeight = metadata.height; // Build SVG overlay const svg = buildAnnotationSvg(imageWidth, imageHeight, annotations); const svgBuffer = Buffer.from(svg); // Composite SVG onto the original image and save as PNG fs.mkdirSync(path.dirname(resolvedOutput), { recursive: true }); await sharp(resolvedInput) .composite([{ input: svgBuffer, top: 0, left: 0 }]) .png() .toFile(resolvedOutput); const count = annotations.length; return { output: `Annotated image saved to ${outputPath} (${count} annotation${count !== 1 ? 's' : ''} applied)`, isError: false, }; } catch (e) { return { output: `AnnotateImage failed: ${(e as Error).message}`, isError: true }; } } export async function executeTool( name: string, input: Record, ctx: ToolContext, ): Promise { switch (name) { case 'ReadImage': return executeReadImage(input, ctx); case 'AnnotateImage': return executeAnnotateImage(input, ctx); default: return null; } }