feat: initial public release (MAESTRO)
This commit is contained in:
oss-sync
@@ -0,0 +1,464 @@
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import sharp from 'sharp';
|
||||
import { ToolDef } from '../../llm/openai-compat.js';
|
||||
import type { ToolContext, ToolResult } from './core.js';
|
||||
import { resolveAndGuard, resolveOutputPathWithin } from './core.js';
|
||||
import { logger } from '../../logger.js';
|
||||
|
||||
// --- Supported image extensions ---
|
||||
|
||||
const SUPPORTED_EXTENSIONS = new Set(['png', 'jpg', 'jpeg', 'gif', 'webp', 'bmp']);
|
||||
const DEFAULT_IMAGE_PROMPT = '画像の内容を日本語で説明してください。';
|
||||
|
||||
// Normalize extension for MIME type (jpg -> jpeg)
|
||||
function extToMime(ext: string): string {
|
||||
if (ext === 'jpg') return 'jpeg';
|
||||
return ext;
|
||||
}
|
||||
|
||||
// --- SVG annotation helpers ---
|
||||
|
||||
interface Annotation {
|
||||
type: 'rectangle' | 'arrow' | 'text';
|
||||
x?: number;
|
||||
y?: number;
|
||||
width?: number;
|
||||
height?: number;
|
||||
from_x?: number;
|
||||
from_y?: number;
|
||||
to_x?: number;
|
||||
to_y?: number;
|
||||
text?: string;
|
||||
color?: string;
|
||||
label?: string;
|
||||
font_size?: number;
|
||||
}
|
||||
|
||||
function computeScaling(imageWidth: number, imageHeight: number): { strokeWidth: number; fontSize: number } {
|
||||
const shortSide = Math.min(imageWidth, imageHeight);
|
||||
return {
|
||||
strokeWidth: Math.max(2, Math.min(5, Math.round(shortSide / 400))),
|
||||
fontSize: Math.max(12, Math.min(32, Math.round(shortSide / 40))),
|
||||
};
|
||||
}
|
||||
|
||||
function escapeXml(str: string): string {
|
||||
return str
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, ''');
|
||||
}
|
||||
|
||||
// Convert a CSS color value into a string safe for use as an SVG `id`.
|
||||
// SVG ids referenced via `url(#...)` may not contain spaces, parens, or commas,
|
||||
// so functional notations like `rgb(255, 0, 0)` cannot be used as-is.
|
||||
function colorToMarkerId(color: string): string {
|
||||
return `ah-${color.replace(/[^a-zA-Z0-9]/g, '_')}`;
|
||||
}
|
||||
|
||||
function buildAnnotationSvg(
|
||||
imageWidth: number,
|
||||
imageHeight: number,
|
||||
annotations: Annotation[],
|
||||
): string {
|
||||
const { strokeWidth, fontSize: defaultFontSize } = computeScaling(imageWidth, imageHeight);
|
||||
const markerSize = strokeWidth * 3;
|
||||
const parts: string[] = [];
|
||||
|
||||
parts.push(`<svg xmlns="http://www.w3.org/2000/svg" width="${imageWidth}" height="${imageHeight}">`);
|
||||
parts.push('<defs>');
|
||||
// One arrowhead marker per unique color
|
||||
const arrowColors = new Set<string>();
|
||||
for (const a of annotations) {
|
||||
if (a.type === 'arrow') arrowColors.add(a.color ?? '#FF0000');
|
||||
}
|
||||
for (const color of arrowColors) {
|
||||
const id = colorToMarkerId(color);
|
||||
parts.push(
|
||||
`<marker id="${id}" markerWidth="${markerSize}" markerHeight="${markerSize * 0.7}" ` +
|
||||
`refX="${markerSize}" refY="${markerSize * 0.35}" orient="auto">` +
|
||||
`<polygon points="0 0, ${markerSize} ${markerSize * 0.35}, 0 ${markerSize * 0.7}" fill="${color}"/>` +
|
||||
'</marker>',
|
||||
);
|
||||
}
|
||||
parts.push('</defs>');
|
||||
|
||||
for (const a of annotations) {
|
||||
const color = a.color ?? '#FF0000';
|
||||
const fontSize = a.font_size ?? defaultFontSize;
|
||||
const labelPadding = Math.round(fontSize * 0.3);
|
||||
|
||||
switch (a.type) {
|
||||
case 'rectangle': {
|
||||
const x = a.x ?? 0;
|
||||
const y = a.y ?? 0;
|
||||
const w = a.width ?? 50;
|
||||
const h = a.height ?? 50;
|
||||
parts.push(
|
||||
`<rect x="${x}" y="${y}" width="${w}" height="${h}" ` +
|
||||
`fill="none" stroke="${color}" stroke-width="${strokeWidth}"/>`,
|
||||
);
|
||||
if (a.label) {
|
||||
const labelWidth = a.label.length * fontSize * 0.6 + labelPadding * 2;
|
||||
const labelHeight = fontSize + labelPadding * 2;
|
||||
const labelY = y - labelHeight;
|
||||
const textY = y - labelPadding;
|
||||
parts.push(
|
||||
`<rect x="${x}" y="${labelY}" width="${labelWidth}" height="${labelHeight}" ` +
|
||||
`fill="${color}" rx="2"/>`,
|
||||
);
|
||||
parts.push(
|
||||
`<text x="${x + labelPadding}" y="${textY}" ` +
|
||||
`fill="white" font-size="${fontSize}" font-family="sans-serif">${escapeXml(a.label)}</text>`,
|
||||
);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'arrow': {
|
||||
const fx = a.from_x ?? 0;
|
||||
const fy = a.from_y ?? 0;
|
||||
const tx = a.to_x ?? 0;
|
||||
const ty = a.to_y ?? 0;
|
||||
const markerId = colorToMarkerId(color);
|
||||
parts.push(
|
||||
`<line x1="${fx}" y1="${fy}" x2="${tx}" y2="${ty}" ` +
|
||||
`stroke="${color}" stroke-width="${strokeWidth}" marker-end="url(#${markerId})"/>`,
|
||||
);
|
||||
if (a.label) {
|
||||
const bgWidth = a.label.length * fontSize * 0.6 + labelPadding * 2;
|
||||
const bgHeight = fontSize + labelPadding * 2;
|
||||
parts.push(
|
||||
`<rect x="${fx}" y="${fy - bgHeight}" width="${bgWidth}" height="${bgHeight}" ` +
|
||||
`fill="rgba(0,0,0,0.6)" rx="3"/>`,
|
||||
);
|
||||
parts.push(
|
||||
`<text x="${fx + labelPadding}" y="${fy - labelPadding}" ` +
|
||||
`fill="${color}" font-size="${fontSize}" font-family="sans-serif">${escapeXml(a.label)}</text>`,
|
||||
);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'text': {
|
||||
const x = a.x ?? 0;
|
||||
const y = a.y ?? 0;
|
||||
const text = a.text ?? '';
|
||||
if (text) {
|
||||
const bgWidth = text.length * fontSize * 0.6 + labelPadding * 2;
|
||||
const bgHeight = fontSize + labelPadding * 2;
|
||||
parts.push(
|
||||
`<rect x="${x - labelPadding}" y="${y - fontSize}" width="${bgWidth}" height="${bgHeight}" ` +
|
||||
`fill="rgba(0,0,0,0.6)" rx="3"/>`,
|
||||
);
|
||||
parts.push(
|
||||
`<text x="${x}" y="${y}" ` +
|
||||
`fill="${color}" font-size="${fontSize}" font-family="sans-serif">${escapeXml(text)}</text>`,
|
||||
);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
parts.push('</svg>');
|
||||
return parts.join('\n');
|
||||
}
|
||||
|
||||
// --- Tool definition ---
|
||||
|
||||
const READIMAGE_DEF: ToolDef = {
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'ReadImage',
|
||||
description: '画像ファイルを LLM に直接渡して内容を認識させる（VLM 対応 worker のみ）。詳細は ReadToolDoc({ name: "ReadImage" })。',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
file_path: { type: 'string', description: 'workspace 内の画像ファイルパス' },
|
||||
prompt: { type: 'string', description: '画像について質問したいこと（省略可）' },
|
||||
},
|
||||
required: ['file_path'],
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const ANNOTATEIMAGE_DEF: ToolDef = {
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'AnnotateImage',
|
||||
description: '画像上に矩形枠・矢印・テキストラベルを描画して output/ に保存する（元画像は変更されない）。詳細は ReadToolDoc({ name: "AnnotateImage" })。',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
input_path: { type: 'string', description: 'workspace 内の元画像ファイルパス' },
|
||||
output_path: { type: 'string', description: '出力先パス（output/ 配下）' },
|
||||
annotations: {
|
||||
type: 'array',
|
||||
description: '描画する注釈の配列',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
type: { type: 'string', enum: ['rectangle', 'arrow', 'text'], description: '注釈の種類' },
|
||||
x: { type: 'number', description: 'rectangle/text: 左上 X 座標' },
|
||||
y: { type: 'number', description: 'rectangle/text: 左上 Y 座標' },
|
||||
width: { type: 'number', description: 'rectangle: 幅' },
|
||||
height: { type: 'number', description: 'rectangle: 高さ' },
|
||||
from_x: { type: 'number', description: 'arrow: 始点 X' },
|
||||
from_y: { type: 'number', description: 'arrow: 始点 Y' },
|
||||
to_x: { type: 'number', description: 'arrow: 終点 X' },
|
||||
to_y: { type: 'number', description: 'arrow: 終点 Y' },
|
||||
text: { type: 'string', description: 'text: 表示するテキスト' },
|
||||
color: { type: 'string', description: '色（デフォルト: #FF0000）' },
|
||||
label: { type: 'string', description: 'rectangle/arrow: ラベルテキスト' },
|
||||
font_size: { type: 'number', description: 'フォントサイズ（省略時は画像サイズから自動算出）' },
|
||||
},
|
||||
required: ['type'],
|
||||
},
|
||||
},
|
||||
},
|
||||
required: ['input_path', 'output_path', 'annotations'],
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
export const TOOL_DEFS: Record<string, ToolDef> = {
|
||||
ReadImage: READIMAGE_DEF,
|
||||
AnnotateImage: ANNOTATEIMAGE_DEF,
|
||||
};
|
||||
|
||||
// --- Tool execution ---
|
||||
|
||||
export function resolveImagePath(
|
||||
filePath: string,
|
||||
ctx: ToolContext,
|
||||
): ToolResult | { resolved: string; ext: string; dataUrl: string } {
|
||||
let resolved: string;
|
||||
try {
|
||||
resolved = resolveAndGuard(ctx.workspacePath, filePath);
|
||||
} catch (e) {
|
||||
return { output: (e as Error).message, isError: true };
|
||||
}
|
||||
|
||||
const ext = path.extname(resolved).replace('.', '').toLowerCase();
|
||||
if (!SUPPORTED_EXTENSIONS.has(ext)) {
|
||||
return {
|
||||
output: `Unsupported image extension: "${ext}". Supported: ${[...SUPPORTED_EXTENSIONS].join(', ')}`,
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
|
||||
let base64data: string;
|
||||
try {
|
||||
const buf = fs.readFileSync(resolved);
|
||||
base64data = buf.toString('base64');
|
||||
} catch (e) {
|
||||
return { output: `Failed to read image: ${(e as Error).message}`, isError: true };
|
||||
}
|
||||
|
||||
const mimeExt = extToMime(ext);
|
||||
return {
|
||||
resolved,
|
||||
ext,
|
||||
dataUrl: `data:image/${mimeExt};base64,${base64data}`,
|
||||
};
|
||||
}
|
||||
|
||||
export async function callVisionModel(
|
||||
dataUrl: string,
|
||||
prompt: string,
|
||||
ctx: ToolContext,
|
||||
): Promise<ToolResult> {
|
||||
const finalPrompt = prompt.trim() || DEFAULT_IMAGE_PROMPT;
|
||||
|
||||
const toolsConfig = ctx.toolsConfig ?? {};
|
||||
const visionModel = toolsConfig.visionModel ?? 'qwen2-vl:8b-instruct';
|
||||
const visionBaseUrl = toolsConfig.visionBaseUrl ?? 'http://10.0.0.10:11434/v1';
|
||||
const visionTimeout = (toolsConfig.visionTimeout ?? 60) * 1000;
|
||||
const visionMaxTokens = toolsConfig.visionMaxTokens ?? 1024;
|
||||
const requestBody = {
|
||||
model: visionModel,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: finalPrompt },
|
||||
{ type: 'image_url', image_url: { url: dataUrl } },
|
||||
],
|
||||
},
|
||||
],
|
||||
max_tokens: visionMaxTokens,
|
||||
};
|
||||
|
||||
logger.debug(`[ReadImage] calling ${visionBaseUrl}/chat/completions model=${visionModel}`);
|
||||
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), visionTimeout);
|
||||
|
||||
try {
|
||||
const response = await fetch(`${visionBaseUrl}/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(requestBody),
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const text = await response.text().catch(() => '');
|
||||
return { output: `Vision API error ${response.status}: ${text}`, isError: true };
|
||||
}
|
||||
|
||||
const json = (await response.json()) as {
|
||||
choices?: Array<{ message?: { content?: string } }>;
|
||||
};
|
||||
|
||||
const content = json.choices?.[0]?.message?.content;
|
||||
if (typeof content !== 'string') {
|
||||
return { output: 'Vision API returned no content', isError: true };
|
||||
}
|
||||
|
||||
return { output: content, isError: false };
|
||||
} catch (e) {
|
||||
if ((e as Error).name === 'AbortError') {
|
||||
return { output: `Vision API timed out after ${visionTimeout / 1000}s`, isError: true };
|
||||
}
|
||||
return { output: `Vision API request failed: ${(e as Error).message}`, isError: true };
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
async function executeReadImage(
|
||||
input: Record<string, unknown>,
|
||||
ctx: ToolContext,
|
||||
): Promise<ToolResult> {
|
||||
if (!ctx.vlmEnabled) {
|
||||
return { output: 'ReadImage requires VLM-enabled worker (vlm: true)', isError: true };
|
||||
}
|
||||
|
||||
const filePath = input['file_path'] as string;
|
||||
const prompt = typeof input['prompt'] === 'string' ? input['prompt'] : '';
|
||||
|
||||
const resolvedImage = resolveImagePath(filePath, ctx);
|
||||
if ('isError' in resolvedImage) return resolvedImage;
|
||||
|
||||
const description = prompt
|
||||
? `画像を読み込みました: ${filePath}\n指示: ${prompt}`
|
||||
: `画像を読み込みました: ${filePath}`;
|
||||
|
||||
return {
|
||||
output: description,
|
||||
isError: false,
|
||||
images: [{ dataUrl: resolvedImage.dataUrl, label: filePath }],
|
||||
};
|
||||
}
|
||||
|
||||
function isValidCssColor(color: string): boolean {
|
||||
if (typeof color !== 'string') return false;
|
||||
// Hex: #RGB, #RRGGBB, #RRGGBBAA
|
||||
if (/^#[0-9A-Fa-f]{3}$|^#[0-9A-Fa-f]{6}$|^#[0-9A-Fa-f]{8}$/.test(color)) return true;
|
||||
// Named colors (lowercase letters only — covers "red", "blue", "transparent", etc.)
|
||||
if (/^[a-z]+$/.test(color)) return true;
|
||||
// rgb(), rgba(), hsl(), hsla() — basic shape only
|
||||
if (/^(?:rgb|rgba|hsl|hsla)\(\s*[\d.,%\s/]+\s*\)$/.test(color)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
const VALID_TYPES = new Set(['rectangle', 'arrow', 'text']);
|
||||
|
||||
async function executeAnnotateImage(
|
||||
input: Record<string, unknown>,
|
||||
ctx: ToolContext,
|
||||
): Promise<ToolResult> {
|
||||
if (!ctx.editAllowed) {
|
||||
return { output: 'AnnotateImage is not allowed: edit flag is false', isError: true };
|
||||
}
|
||||
|
||||
const inputPath = input['input_path'] as string;
|
||||
const outputPath = input['output_path'] as string;
|
||||
const annotations = input['annotations'] as Annotation[];
|
||||
|
||||
if (!Array.isArray(annotations) || annotations.length === 0) {
|
||||
return { output: 'AnnotateImage requires at least one annotation', isError: true };
|
||||
}
|
||||
|
||||
for (let i = 0; i < annotations.length; i++) {
|
||||
const a = annotations[i];
|
||||
if (!VALID_TYPES.has(a.type)) {
|
||||
return {
|
||||
output: `AnnotateImage failed: annotation[${i}].type "${a.type}" is invalid. Must be one of: rectangle, arrow, text.`,
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
if (a.color !== undefined && !isValidCssColor(a.color)) {
|
||||
return {
|
||||
output: `AnnotateImage failed: annotation[${i}] has invalid color "${a.color}". Use formats like "#FF0000", "red", or "rgb(255,0,0)".`,
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve and guard input path
|
||||
let resolvedInput: string;
|
||||
try {
|
||||
resolvedInput = resolveAndGuard(ctx.workspacePath, inputPath);
|
||||
} catch (e) {
|
||||
return { output: (e as Error).message, isError: true };
|
||||
}
|
||||
|
||||
// Resolve and guard output path (must be within output/)
|
||||
let resolvedOutput: string;
|
||||
try {
|
||||
resolvedOutput = resolveOutputPathWithin(ctx.workspacePath, outputPath, ['output']);
|
||||
} catch (e) {
|
||||
return { output: (e as Error).message, isError: true };
|
||||
}
|
||||
|
||||
try {
|
||||
// Read image metadata
|
||||
const metadata = await sharp(resolvedInput).metadata();
|
||||
if (!metadata.width || !metadata.height) {
|
||||
return {
|
||||
output: `AnnotateImage failed: cannot determine image dimensions for ${inputPath}`,
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
const imageWidth = metadata.width;
|
||||
const imageHeight = metadata.height;
|
||||
|
||||
// Build SVG overlay
|
||||
const svg = buildAnnotationSvg(imageWidth, imageHeight, annotations);
|
||||
const svgBuffer = Buffer.from(svg);
|
||||
|
||||
// Composite SVG onto the original image and save as PNG
|
||||
fs.mkdirSync(path.dirname(resolvedOutput), { recursive: true });
|
||||
await sharp(resolvedInput)
|
||||
.composite([{ input: svgBuffer, top: 0, left: 0 }])
|
||||
.png()
|
||||
.toFile(resolvedOutput);
|
||||
|
||||
const count = annotations.length;
|
||||
return {
|
||||
output: `Annotated image saved to ${outputPath} (${count} annotation${count !== 1 ? 's' : ''} applied)`,
|
||||
isError: false,
|
||||
};
|
||||
} catch (e) {
|
||||
return { output: `AnnotateImage failed: ${(e as Error).message}`, isError: true };
|
||||
}
|
||||
}
|
||||
|
||||
export async function executeTool(
|
||||
name: string,
|
||||
input: Record<string, unknown>,
|
||||
ctx: ToolContext,
|
||||
): Promise<ToolResult | null> {
|
||||
switch (name) {
|
||||
case 'ReadImage':
|
||||
return executeReadImage(input, ctx);
|
||||
case 'AnnotateImage':
|
||||
return executeAnnotateImage(input, ctx);
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user