484 lines
16 KiB
TypeScript
484 lines
16 KiB
TypeScript
import * as fs from 'fs';
|
||
import * as path from 'path';
|
||
import sharp from 'sharp';
|
||
import { ToolDef } from '../../llm/openai-compat.js';
|
||
import type { ToolContext, ToolResult } from './core.js';
|
||
import { resolveAndGuard, resolveOutputPathWithin } from './core.js';
|
||
import { logger } from '../../logger.js';
|
||
import { recordLlmUsage } from '../../llm/usage-recorder.js';
|
||
|
||
// --- Supported image extensions ---
|
||
|
||
const SUPPORTED_EXTENSIONS = new Set(['png', 'jpg', 'jpeg', 'gif', 'webp', 'bmp']);
|
||
const DEFAULT_IMAGE_PROMPT = '画像の内容を日本語で説明してください。';
|
||
|
||
// Normalize extension for MIME type (jpg -> jpeg)
|
||
function extToMime(ext: string): string {
|
||
if (ext === 'jpg') return 'jpeg';
|
||
return ext;
|
||
}
|
||
|
||
// --- SVG annotation helpers ---
|
||
|
||
interface Annotation {
|
||
type: 'rectangle' | 'arrow' | 'text';
|
||
x?: number;
|
||
y?: number;
|
||
width?: number;
|
||
height?: number;
|
||
from_x?: number;
|
||
from_y?: number;
|
||
to_x?: number;
|
||
to_y?: number;
|
||
text?: string;
|
||
color?: string;
|
||
label?: string;
|
||
font_size?: number;
|
||
}
|
||
|
||
function computeScaling(imageWidth: number, imageHeight: number): { strokeWidth: number; fontSize: number } {
|
||
const shortSide = Math.min(imageWidth, imageHeight);
|
||
return {
|
||
strokeWidth: Math.max(2, Math.min(5, Math.round(shortSide / 400))),
|
||
fontSize: Math.max(12, Math.min(32, Math.round(shortSide / 40))),
|
||
};
|
||
}
|
||
|
||
function escapeXml(str: string): string {
|
||
return str
|
||
.replace(/&/g, '&')
|
||
.replace(/</g, '<')
|
||
.replace(/>/g, '>')
|
||
.replace(/"/g, '"')
|
||
.replace(/'/g, ''');
|
||
}
|
||
|
||
// Convert a CSS color value into a string safe for use as an SVG `id`.
|
||
// SVG ids referenced via `url(#...)` may not contain spaces, parens, or commas,
|
||
// so functional notations like `rgb(255, 0, 0)` cannot be used as-is.
|
||
function colorToMarkerId(color: string): string {
|
||
return `ah-${color.replace(/[^a-zA-Z0-9]/g, '_')}`;
|
||
}
|
||
|
||
function buildAnnotationSvg(
|
||
imageWidth: number,
|
||
imageHeight: number,
|
||
annotations: Annotation[],
|
||
): string {
|
||
const { strokeWidth, fontSize: defaultFontSize } = computeScaling(imageWidth, imageHeight);
|
||
const markerSize = strokeWidth * 3;
|
||
const parts: string[] = [];
|
||
|
||
parts.push(`<svg xmlns="http://www.w3.org/2000/svg" width="${imageWidth}" height="${imageHeight}">`);
|
||
parts.push('<defs>');
|
||
// One arrowhead marker per unique color
|
||
const arrowColors = new Set<string>();
|
||
for (const a of annotations) {
|
||
if (a.type === 'arrow') arrowColors.add(a.color ?? '#FF0000');
|
||
}
|
||
for (const color of arrowColors) {
|
||
const id = colorToMarkerId(color);
|
||
parts.push(
|
||
`<marker id="${id}" markerWidth="${markerSize}" markerHeight="${markerSize * 0.7}" ` +
|
||
`refX="${markerSize}" refY="${markerSize * 0.35}" orient="auto">` +
|
||
`<polygon points="0 0, ${markerSize} ${markerSize * 0.35}, 0 ${markerSize * 0.7}" fill="${color}"/>` +
|
||
'</marker>',
|
||
);
|
||
}
|
||
parts.push('</defs>');
|
||
|
||
for (const a of annotations) {
|
||
const color = a.color ?? '#FF0000';
|
||
const fontSize = a.font_size ?? defaultFontSize;
|
||
const labelPadding = Math.round(fontSize * 0.3);
|
||
|
||
switch (a.type) {
|
||
case 'rectangle': {
|
||
const x = a.x ?? 0;
|
||
const y = a.y ?? 0;
|
||
const w = a.width ?? 50;
|
||
const h = a.height ?? 50;
|
||
parts.push(
|
||
`<rect x="${x}" y="${y}" width="${w}" height="${h}" ` +
|
||
`fill="none" stroke="${color}" stroke-width="${strokeWidth}"/>`,
|
||
);
|
||
if (a.label) {
|
||
const labelWidth = a.label.length * fontSize * 0.6 + labelPadding * 2;
|
||
const labelHeight = fontSize + labelPadding * 2;
|
||
const labelY = y - labelHeight;
|
||
const textY = y - labelPadding;
|
||
parts.push(
|
||
`<rect x="${x}" y="${labelY}" width="${labelWidth}" height="${labelHeight}" ` +
|
||
`fill="${color}" rx="2"/>`,
|
||
);
|
||
parts.push(
|
||
`<text x="${x + labelPadding}" y="${textY}" ` +
|
||
`fill="white" font-size="${fontSize}" font-family="sans-serif">${escapeXml(a.label)}</text>`,
|
||
);
|
||
}
|
||
break;
|
||
}
|
||
case 'arrow': {
|
||
const fx = a.from_x ?? 0;
|
||
const fy = a.from_y ?? 0;
|
||
const tx = a.to_x ?? 0;
|
||
const ty = a.to_y ?? 0;
|
||
const markerId = colorToMarkerId(color);
|
||
parts.push(
|
||
`<line x1="${fx}" y1="${fy}" x2="${tx}" y2="${ty}" ` +
|
||
`stroke="${color}" stroke-width="${strokeWidth}" marker-end="url(#${markerId})"/>`,
|
||
);
|
||
if (a.label) {
|
||
const bgWidth = a.label.length * fontSize * 0.6 + labelPadding * 2;
|
||
const bgHeight = fontSize + labelPadding * 2;
|
||
parts.push(
|
||
`<rect x="${fx}" y="${fy - bgHeight}" width="${bgWidth}" height="${bgHeight}" ` +
|
||
`fill="rgba(0,0,0,0.6)" rx="3"/>`,
|
||
);
|
||
parts.push(
|
||
`<text x="${fx + labelPadding}" y="${fy - labelPadding}" ` +
|
||
`fill="${color}" font-size="${fontSize}" font-family="sans-serif">${escapeXml(a.label)}</text>`,
|
||
);
|
||
}
|
||
break;
|
||
}
|
||
case 'text': {
|
||
const x = a.x ?? 0;
|
||
const y = a.y ?? 0;
|
||
const text = a.text ?? '';
|
||
if (text) {
|
||
const bgWidth = text.length * fontSize * 0.6 + labelPadding * 2;
|
||
const bgHeight = fontSize + labelPadding * 2;
|
||
parts.push(
|
||
`<rect x="${x - labelPadding}" y="${y - fontSize}" width="${bgWidth}" height="${bgHeight}" ` +
|
||
`fill="rgba(0,0,0,0.6)" rx="3"/>`,
|
||
);
|
||
parts.push(
|
||
`<text x="${x}" y="${y}" ` +
|
||
`fill="${color}" font-size="${fontSize}" font-family="sans-serif">${escapeXml(text)}</text>`,
|
||
);
|
||
}
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
|
||
parts.push('</svg>');
|
||
return parts.join('\n');
|
||
}
|
||
|
||
// --- Tool definition ---
|
||
|
||
const READIMAGE_DEF: ToolDef = {
|
||
type: 'function',
|
||
function: {
|
||
name: 'ReadImage',
|
||
description: '画像ファイルを LLM に直接渡して内容を認識させる（VLM 対応 worker のみ）。詳細は ReadToolDoc({ name: "ReadImage" })。',
|
||
parameters: {
|
||
type: 'object',
|
||
properties: {
|
||
file_path: { type: 'string', description: 'workspace 内の画像ファイルパス' },
|
||
prompt: { type: 'string', description: '画像について質問したいこと（省略可）' },
|
||
},
|
||
required: ['file_path'],
|
||
},
|
||
},
|
||
};
|
||
|
||
const ANNOTATEIMAGE_DEF: ToolDef = {
|
||
type: 'function',
|
||
function: {
|
||
name: 'AnnotateImage',
|
||
description: '画像上に矩形枠・矢印・テキストラベルを描画して output/ に保存する（元画像は変更されない）。詳細は ReadToolDoc({ name: "AnnotateImage" })。',
|
||
parameters: {
|
||
type: 'object',
|
||
properties: {
|
||
input_path: { type: 'string', description: 'workspace 内の元画像ファイルパス' },
|
||
output_path: { type: 'string', description: '出力先パス（output/ 配下）' },
|
||
annotations: {
|
||
type: 'array',
|
||
description: '描画する注釈の配列',
|
||
items: {
|
||
type: 'object',
|
||
properties: {
|
||
type: { type: 'string', enum: ['rectangle', 'arrow', 'text'], description: '注釈の種類' },
|
||
x: { type: 'number', description: 'rectangle/text: 左上 X 座標' },
|
||
y: { type: 'number', description: 'rectangle/text: 左上 Y 座標' },
|
||
width: { type: 'number', description: 'rectangle: 幅' },
|
||
height: { type: 'number', description: 'rectangle: 高さ' },
|
||
from_x: { type: 'number', description: 'arrow: 始点 X' },
|
||
from_y: { type: 'number', description: 'arrow: 始点 Y' },
|
||
to_x: { type: 'number', description: 'arrow: 終点 X' },
|
||
to_y: { type: 'number', description: 'arrow: 終点 Y' },
|
||
text: { type: 'string', description: 'text: 表示するテキスト' },
|
||
color: { type: 'string', description: '色（デフォルト: #FF0000）' },
|
||
label: { type: 'string', description: 'rectangle/arrow: ラベルテキスト' },
|
||
font_size: { type: 'number', description: 'フォントサイズ（省略時は画像サイズから自動算出）' },
|
||
},
|
||
required: ['type'],
|
||
},
|
||
},
|
||
},
|
||
required: ['input_path', 'output_path', 'annotations'],
|
||
},
|
||
},
|
||
};
|
||
|
||
export const TOOL_DEFS: Record<string, ToolDef> = {
|
||
ReadImage: READIMAGE_DEF,
|
||
AnnotateImage: ANNOTATEIMAGE_DEF,
|
||
};
|
||
|
||
// --- Tool execution ---
|
||
|
||
export function resolveImagePath(
|
||
filePath: string,
|
||
ctx: ToolContext,
|
||
): ToolResult | { resolved: string; ext: string; dataUrl: string } {
|
||
let resolved: string;
|
||
try {
|
||
resolved = resolveAndGuard(ctx.workspacePath, filePath);
|
||
} catch (e) {
|
||
return { output: (e as Error).message, isError: true };
|
||
}
|
||
|
||
const ext = path.extname(resolved).replace('.', '').toLowerCase();
|
||
if (!SUPPORTED_EXTENSIONS.has(ext)) {
|
||
return {
|
||
output: `Unsupported image extension: "${ext}". Supported: ${[...SUPPORTED_EXTENSIONS].join(', ')}`,
|
||
isError: true,
|
||
};
|
||
}
|
||
|
||
let base64data: string;
|
||
try {
|
||
const buf = fs.readFileSync(resolved);
|
||
base64data = buf.toString('base64');
|
||
} catch (e) {
|
||
return { output: `Failed to read image: ${(e as Error).message}`, isError: true };
|
||
}
|
||
|
||
const mimeExt = extToMime(ext);
|
||
return {
|
||
resolved,
|
||
ext,
|
||
dataUrl: `data:image/${mimeExt};base64,${base64data}`,
|
||
};
|
||
}
|
||
|
||
export async function callVisionModel(
|
||
dataUrl: string,
|
||
prompt: string,
|
||
ctx: ToolContext,
|
||
): Promise<ToolResult> {
|
||
const finalPrompt = prompt.trim() || DEFAULT_IMAGE_PROMPT;
|
||
|
||
const toolsConfig = ctx.toolsConfig ?? {};
|
||
const visionModel = toolsConfig.visionModel ?? 'qwen2-vl:8b-instruct';
|
||
// Neutral placeholder — set tools.vision_base_url in config.yaml for a real endpoint.
|
||
// (Workers usually override this with their own endpoint; see worker.ts)
|
||
const visionBaseUrl = toolsConfig.visionBaseUrl ?? 'http://10.0.0.10:11434/v1';
|
||
const visionTimeout = (toolsConfig.visionTimeout ?? 60) * 1000;
|
||
const visionMaxTokens = toolsConfig.visionMaxTokens ?? 1024;
|
||
const requestBody = {
|
||
model: visionModel,
|
||
messages: [
|
||
{
|
||
role: 'user',
|
||
content: [
|
||
{ type: 'text', text: finalPrompt },
|
||
{ type: 'image_url', image_url: { url: dataUrl } },
|
||
],
|
||
},
|
||
],
|
||
max_tokens: visionMaxTokens,
|
||
};
|
||
|
||
logger.debug(`[ReadImage] calling ${visionBaseUrl}/chat/completions model=${visionModel}`);
|
||
|
||
const controller = new AbortController();
|
||
const timer = setTimeout(() => controller.abort(), visionTimeout);
|
||
|
||
try {
|
||
const response = await fetch(`${visionBaseUrl}/chat/completions`, {
|
||
method: 'POST',
|
||
headers: { 'Content-Type': 'application/json' },
|
||
body: JSON.stringify(requestBody),
|
||
signal: controller.signal,
|
||
});
|
||
|
||
if (!response.ok) {
|
||
const text = await response.text().catch(() => '');
|
||
return { output: `Vision API error ${response.status}: ${text}`, isError: true };
|
||
}
|
||
|
||
const json = (await response.json()) as {
|
||
model?: string;
|
||
choices?: Array<{ message?: { content?: string } }>;
|
||
usage?: { prompt_tokens?: number; completion_tokens?: number };
|
||
};
|
||
|
||
const content = json.choices?.[0]?.message?.content;
|
||
if (typeof content !== 'string') {
|
||
return { output: 'Vision API returned no content', isError: true };
|
||
}
|
||
|
||
// This vision call is a raw fetch outside OpenAICompatClient, so it would
|
||
// otherwise miss the per-user usage ledger. Record it directly (issue
|
||
// #499). Vision uses its own endpoint directly (never the gateway).
|
||
let route = 'unknown';
|
||
try { route = new URL(visionBaseUrl).host || 'unknown'; } catch { /* keep 'unknown' */ }
|
||
recordLlmUsage({
|
||
userId: ctx.userId ?? 'local',
|
||
source: 'direct',
|
||
model: json.model || visionModel,
|
||
route,
|
||
tokensIn: json.usage?.prompt_tokens ?? 0,
|
||
tokensOut: json.usage?.completion_tokens ?? 0,
|
||
});
|
||
|
||
return { output: content, isError: false };
|
||
} catch (e) {
|
||
if ((e as Error).name === 'AbortError') {
|
||
return { output: `Vision API timed out after ${visionTimeout / 1000}s`, isError: true };
|
||
}
|
||
return { output: `Vision API request failed: ${(e as Error).message}`, isError: true };
|
||
} finally {
|
||
clearTimeout(timer);
|
||
}
|
||
}
|
||
|
||
async function executeReadImage(
|
||
input: Record<string, unknown>,
|
||
ctx: ToolContext,
|
||
): Promise<ToolResult> {
|
||
if (!ctx.vlmEnabled) {
|
||
return { output: 'ReadImage requires VLM-enabled worker (vlm: true)', isError: true };
|
||
}
|
||
|
||
const filePath = input['file_path'] as string;
|
||
const prompt = typeof input['prompt'] === 'string' ? input['prompt'] : '';
|
||
|
||
const resolvedImage = resolveImagePath(filePath, ctx);
|
||
if ('isError' in resolvedImage) return resolvedImage;
|
||
|
||
const description = prompt
|
||
? `画像を読み込みました: ${filePath}\n指示: ${prompt}`
|
||
: `画像を読み込みました: ${filePath}`;
|
||
|
||
return {
|
||
output: description,
|
||
isError: false,
|
||
images: [{ dataUrl: resolvedImage.dataUrl, label: filePath }],
|
||
};
|
||
}
|
||
|
||
function isValidCssColor(color: string): boolean {
|
||
if (typeof color !== 'string') return false;
|
||
// Hex: #RGB, #RRGGBB, #RRGGBBAA
|
||
if (/^#[0-9A-Fa-f]{3}$|^#[0-9A-Fa-f]{6}$|^#[0-9A-Fa-f]{8}$/.test(color)) return true;
|
||
// Named colors (lowercase letters only — covers "red", "blue", "transparent", etc.)
|
||
if (/^[a-z]+$/.test(color)) return true;
|
||
// rgb(), rgba(), hsl(), hsla() — basic shape only
|
||
if (/^(?:rgb|rgba|hsl|hsla)\(\s*[\d.,%\s/]+\s*\)$/.test(color)) return true;
|
||
return false;
|
||
}
|
||
|
||
const VALID_TYPES = new Set(['rectangle', 'arrow', 'text']);
|
||
|
||
async function executeAnnotateImage(
|
||
input: Record<string, unknown>,
|
||
ctx: ToolContext,
|
||
): Promise<ToolResult> {
|
||
if (!ctx.editAllowed) {
|
||
return { output: 'AnnotateImage is not allowed: edit flag is false', isError: true };
|
||
}
|
||
|
||
const inputPath = input['input_path'] as string;
|
||
const outputPath = input['output_path'] as string;
|
||
const annotations = input['annotations'] as Annotation[];
|
||
|
||
if (!Array.isArray(annotations) || annotations.length === 0) {
|
||
return { output: 'AnnotateImage requires at least one annotation', isError: true };
|
||
}
|
||
|
||
for (let i = 0; i < annotations.length; i++) {
|
||
const a = annotations[i];
|
||
if (!VALID_TYPES.has(a.type)) {
|
||
return {
|
||
output: `AnnotateImage failed: annotation[${i}].type "${a.type}" is invalid. Must be one of: rectangle, arrow, text.`,
|
||
isError: true,
|
||
};
|
||
}
|
||
if (a.color !== undefined && !isValidCssColor(a.color)) {
|
||
return {
|
||
output: `AnnotateImage failed: annotation[${i}] has invalid color "${a.color}". Use formats like "#FF0000", "red", or "rgb(255,0,0)".`,
|
||
isError: true,
|
||
};
|
||
}
|
||
}
|
||
|
||
// Resolve and guard input path
|
||
let resolvedInput: string;
|
||
try {
|
||
resolvedInput = resolveAndGuard(ctx.workspacePath, inputPath);
|
||
} catch (e) {
|
||
return { output: (e as Error).message, isError: true };
|
||
}
|
||
|
||
// Resolve and guard output path (must be within output/)
|
||
let resolvedOutput: string;
|
||
try {
|
||
resolvedOutput = resolveOutputPathWithin(ctx.workspacePath, outputPath, ['output']);
|
||
} catch (e) {
|
||
return { output: (e as Error).message, isError: true };
|
||
}
|
||
|
||
try {
|
||
// Read image metadata
|
||
const metadata = await sharp(resolvedInput).metadata();
|
||
if (!metadata.width || !metadata.height) {
|
||
return {
|
||
output: `AnnotateImage failed: cannot determine image dimensions for ${inputPath}`,
|
||
isError: true,
|
||
};
|
||
}
|
||
const imageWidth = metadata.width;
|
||
const imageHeight = metadata.height;
|
||
|
||
// Build SVG overlay
|
||
const svg = buildAnnotationSvg(imageWidth, imageHeight, annotations);
|
||
const svgBuffer = Buffer.from(svg);
|
||
|
||
// Composite SVG onto the original image and save as PNG
|
||
fs.mkdirSync(path.dirname(resolvedOutput), { recursive: true });
|
||
await sharp(resolvedInput)
|
||
.composite([{ input: svgBuffer, top: 0, left: 0 }])
|
||
.png()
|
||
.toFile(resolvedOutput);
|
||
|
||
const count = annotations.length;
|
||
return {
|
||
output: `Annotated image saved to ${outputPath} (${count} annotation${count !== 1 ? 's' : ''} applied)`,
|
||
isError: false,
|
||
};
|
||
} catch (e) {
|
||
return { output: `AnnotateImage failed: ${(e as Error).message}`, isError: true };
|
||
}
|
||
}
|
||
|
||
export async function executeTool(
|
||
name: string,
|
||
input: Record<string, unknown>,
|
||
ctx: ToolContext,
|
||
): Promise<ToolResult | null> {
|
||
switch (name) {
|
||
case 'ReadImage':
|
||
return executeReadImage(input, ctx);
|
||
case 'AnnotateImage':
|
||
return executeAnnotateImage(input, ctx);
|
||
default:
|
||
return null;
|
||
}
|
||
}
|