feat: initial public release (MAESTRO)
This commit is contained in:
oss-sync
@@ -0,0 +1,477 @@
|
||||
// knowledge.ts — DKS (Document Knowledge Service) client tools
|
||||
import { readFileSync, mkdirSync, writeFileSync, appendFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { ToolDef } from '../../llm/openai-compat.js';
|
||||
import type { ToolContext, ToolResult } from './core.js';
|
||||
import { logger } from '../../logger.js';
|
||||
import { generateRawFilename } from './raw-save.js';
|
||||
|
||||
// --- Config access ---
|
||||
|
||||
interface KnowledgeNamespaceConfig {
|
||||
apiKey: string;
|
||||
}
|
||||
|
||||
function getServiceUrl(ctx: ToolContext): string | null {
|
||||
return ctx.toolsConfig?.knowledgeServiceUrl ?? null;
|
||||
}
|
||||
|
||||
function getNamespaces(ctx: ToolContext): Record<string, KnowledgeNamespaceConfig> | null {
|
||||
return ctx.toolsConfig?.knowledgeNamespaces ?? null;
|
||||
}
|
||||
|
||||
function getApiKey(ctx: ToolContext, namespace: string): string | null {
|
||||
return ctx.toolsConfig?.knowledgeNamespaces?.[namespace]?.apiKey ?? null;
|
||||
}
|
||||
|
||||
// --- Fetch helper ---
|
||||
|
||||
async function dksFetch(
|
||||
serviceUrl: string,
|
||||
path: string,
|
||||
apiKey: string,
|
||||
options: RequestInit & { timeoutMs?: number } = {},
|
||||
): Promise<Response> {
|
||||
const url = `${serviceUrl.replace(/\/+$/, '')}${path}`;
|
||||
const timeoutMs = options.timeoutMs ?? 10000;
|
||||
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
|
||||
try {
|
||||
const { timeoutMs: _, ...fetchOptions } = options;
|
||||
const headers: Record<string, string> = {
|
||||
'Authorization': `Bearer ${apiKey}`,
|
||||
...(fetchOptions.headers as Record<string, string> || {}),
|
||||
};
|
||||
|
||||
return await fetch(url, {
|
||||
...fetchOptions,
|
||||
headers,
|
||||
signal: controller.signal,
|
||||
});
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
// --- History logging ---
|
||||
|
||||
interface KnowledgeHistoryRecord {
|
||||
timestamp: string;
|
||||
tool: string;
|
||||
namespace?: string;
|
||||
query?: string;
|
||||
sectionsFound?: number;
|
||||
imagesDownloaded?: number;
|
||||
error?: string;
|
||||
durationMs?: number;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
function appendKnowledgeHistory(ctx: ToolContext, record: KnowledgeHistoryRecord): void {
|
||||
try {
|
||||
const logsDir = join(ctx.workspacePath, 'logs');
|
||||
const historyPath = join(logsDir, 'knowledge-history.jsonl');
|
||||
mkdirSync(logsDir, { recursive: true });
|
||||
appendFileSync(historyPath, `${JSON.stringify(record)}\n`, 'utf-8');
|
||||
} catch (e) {
|
||||
logger.warn(`[knowledge] failed to write history: ${(e as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// --- Raw response save helper ---
|
||||
|
||||
function saveRawResponse(ctx: ToolContext, toolName: string, data: unknown): void {
|
||||
try {
|
||||
const rawDir = join(ctx.workspacePath, 'logs', 'raw');
|
||||
mkdirSync(rawDir, { recursive: true });
|
||||
const filename = generateRawFilename(toolName, '.json');
|
||||
writeFileSync(join(rawDir, filename), JSON.stringify(data, null, 2), 'utf-8');
|
||||
|
||||
const indexPath = join(ctx.workspacePath, 'logs', 'rawdata-history.jsonl');
|
||||
const content = JSON.stringify(data);
|
||||
appendFileSync(indexPath, JSON.stringify({
|
||||
timestamp: new Date().toISOString(),
|
||||
tool: toolName,
|
||||
filename,
|
||||
bytes: Buffer.byteLength(content, 'utf-8'),
|
||||
}) + '\n', 'utf-8');
|
||||
} catch (e) {
|
||||
logger.warn(`[knowledge] failed to save raw response: ${(e as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// --- Page image download helper ---
|
||||
|
||||
async function downloadPageImages(
|
||||
serviceUrl: string,
|
||||
apiKey: string,
|
||||
imageUrls: string[],
|
||||
ctx: ToolContext,
|
||||
namespace: string,
|
||||
): Promise<{ localPaths: string[]; errors: string[] }> {
|
||||
const localPaths: string[] = [];
|
||||
const errors: string[] = [];
|
||||
if (imageUrls.length === 0) return { localPaths, errors };
|
||||
|
||||
const saveDir = join(ctx.workspacePath, 'input', 'knowledge', namespace);
|
||||
mkdirSync(saveDir, { recursive: true });
|
||||
|
||||
for (const relUrl of imageUrls) {
|
||||
const fullUrl = `${serviceUrl.replace(/\/+$/, '')}${relUrl}`;
|
||||
try {
|
||||
const resp = await dksFetch(serviceUrl, relUrl, apiKey, { timeoutMs: 15000 });
|
||||
if (!resp.ok) {
|
||||
errors.push(`${relUrl}: HTTP ${resp.status}`);
|
||||
continue;
|
||||
}
|
||||
const buffer = Buffer.from(await resp.arrayBuffer());
|
||||
// Extract filename from URL path (e.g. /pages/abc123.png -> abc123.png)
|
||||
const urlPath = relUrl.split('/').pop() || `page-${Date.now()}.png`;
|
||||
const localPath = join(saveDir, urlPath);
|
||||
writeFileSync(localPath, buffer);
|
||||
localPaths.push(`input/knowledge/${namespace}/${urlPath}`);
|
||||
} catch (e) {
|
||||
const msg = (e as Error).name === 'AbortError' ? 'timeout' : (e as Error).message;
|
||||
errors.push(`${fullUrl}: ${msg}`);
|
||||
}
|
||||
}
|
||||
return { localPaths, errors };
|
||||
}
|
||||
|
||||
// --- Tool Definitions ---
|
||||
|
||||
const INGEST_DOCUMENT_DEF: ToolDef = {
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'IngestDocument',
|
||||
description: 'ドキュメントをナレッジベースに取り込む（非同期）。PDF, Word, PowerPoint, Excel, 画像, CSV/TSV に対応。',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
namespace: { type: 'string', description: '対象ネームスペース' },
|
||||
file_path: { type: 'string', description: 'ワークスペース内のファイルパス' },
|
||||
},
|
||||
required: ['namespace', 'file_path'],
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const INGEST_STATUS_DEF: ToolDef = {
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'IngestStatus',
|
||||
description: '取込ジョブの進捗状況を確認する。',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
namespace: { type: 'string', description: '対象ネームスペース' },
|
||||
job_id: { type: 'string', description: 'IngestDocument で返却されたジョブID' },
|
||||
},
|
||||
required: ['namespace', 'job_id'],
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const SEARCH_KNOWLEDGE_DEF: ToolDef = {
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'SearchKnowledge',
|
||||
description: 'DKS（社内ナレッジ）を自然言語で検索する。関連セクション（テキスト）+ ページ画像が返り、画像は input/knowledge/{ns}/ に自動保存され ReadImage で閲覧可能。詳細は ReadToolDoc({ name: "SearchKnowledge" })。',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
namespace: { type: 'string', description: '検索対象ネームスペース' },
|
||||
query: { type: 'string', description: '検索クエリ（自然言語）' },
|
||||
},
|
||||
required: ['namespace', 'query'],
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const LIST_NAMESPACES_DEF: ToolDef = {
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'ListNamespaces',
|
||||
description: '利用可能なナレッジベースのネームスペース一覧を表示する。',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {},
|
||||
required: [],
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const LIST_DOCUMENTS_DEF: ToolDef = {
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'ListDocuments',
|
||||
description: 'ネームスペース内の文書一覧を表示する。',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
namespace: { type: 'string', description: '対象ネームスペース' },
|
||||
},
|
||||
required: ['namespace'],
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
export const TOOL_DEFS: Record<string, ToolDef> = {
|
||||
IngestDocument: INGEST_DOCUMENT_DEF,
|
||||
IngestStatus: INGEST_STATUS_DEF,
|
||||
SearchKnowledge: SEARCH_KNOWLEDGE_DEF,
|
||||
ListNamespaces: LIST_NAMESPACES_DEF,
|
||||
ListDocuments: LIST_DOCUMENTS_DEF,
|
||||
};
|
||||
|
||||
// --- Tool Execution ---
|
||||
|
||||
export async function executeTool(
|
||||
name: string,
|
||||
input: Record<string, unknown>,
|
||||
ctx: ToolContext,
|
||||
): Promise<ToolResult | null> {
|
||||
if (name === 'IngestDocument') return executeIngestDocument(input, ctx);
|
||||
if (name === 'IngestStatus') return executeIngestStatus(input, ctx);
|
||||
if (name === 'SearchKnowledge') return executeSearchKnowledge(input, ctx);
|
||||
if (name === 'ListNamespaces') return executeListNamespaces(ctx);
|
||||
if (name === 'ListDocuments') return executeListDocuments(input, ctx);
|
||||
return null;
|
||||
}
|
||||
|
||||
// --- Tool Implementations ---
|
||||
|
||||
async function executeIngestDocument(input: Record<string, unknown>, ctx: ToolContext): Promise<ToolResult> {
|
||||
const serviceUrl = getServiceUrl(ctx);
|
||||
if (!serviceUrl) return { output: 'Knowledge service not configured', isError: true };
|
||||
|
||||
const namespace = input.namespace as string;
|
||||
const filePath = input.file_path as string;
|
||||
const apiKey = getApiKey(ctx, namespace);
|
||||
if (!apiKey) return { output: `Namespace "${namespace}" not configured`, isError: true };
|
||||
|
||||
const startMs = Date.now();
|
||||
try {
|
||||
const resolvedPath = filePath.startsWith('/') ? filePath : `${ctx.workspacePath}/${filePath}`;
|
||||
const fileData = readFileSync(resolvedPath);
|
||||
const fileName = resolvedPath.split('/').pop() || 'unknown';
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append('file', new Blob([fileData]), fileName);
|
||||
|
||||
const resp = await dksFetch(serviceUrl, `/namespaces/${namespace}/ingest`, apiKey, {
|
||||
method: 'POST',
|
||||
body: formData,
|
||||
});
|
||||
|
||||
if (!resp.ok) {
|
||||
const errText = await resp.text();
|
||||
const output = `Ingest failed (${resp.status}): ${errText}`;
|
||||
appendKnowledgeHistory(ctx, {
|
||||
timestamp: new Date().toISOString(), tool: 'IngestDocument',
|
||||
namespace, error: output, durationMs: Date.now() - startMs,
|
||||
});
|
||||
return { output, isError: true };
|
||||
}
|
||||
|
||||
const data = await resp.json() as any;
|
||||
const output = `取込を開始しました (job: ${data.job_id}, ${data.pages_detected}ページ検出)。完了確認は IngestStatus で可能です。他の作業を続行できます。`;
|
||||
appendKnowledgeHistory(ctx, {
|
||||
timestamp: new Date().toISOString(), tool: 'IngestDocument',
|
||||
namespace, fileName, jobId: data.job_id, pagesDetected: data.pages_detected,
|
||||
durationMs: Date.now() - startMs,
|
||||
});
|
||||
return { output, isError: false };
|
||||
} catch (e: any) {
|
||||
const error = e.name === 'AbortError'
|
||||
? `IngestDocument timeout: DKS server did not respond within 10s`
|
||||
: `IngestDocument error: ${e.message}`;
|
||||
appendKnowledgeHistory(ctx, {
|
||||
timestamp: new Date().toISOString(), tool: 'IngestDocument',
|
||||
namespace, error, durationMs: Date.now() - startMs,
|
||||
});
|
||||
return { output: error, isError: true };
|
||||
}
|
||||
}
|
||||
|
||||
async function executeIngestStatus(input: Record<string, unknown>, ctx: ToolContext): Promise<ToolResult> {
|
||||
const serviceUrl = getServiceUrl(ctx);
|
||||
if (!serviceUrl) return { output: 'Knowledge service not configured', isError: true };
|
||||
|
||||
const namespace = input.namespace as string;
|
||||
const jobId = input.job_id as string;
|
||||
const apiKey = getApiKey(ctx, namespace);
|
||||
if (!apiKey) return { output: `Namespace "${namespace}" not configured`, isError: true };
|
||||
|
||||
try {
|
||||
const resp = await dksFetch(serviceUrl, `/namespaces/${namespace}/jobs/${jobId}`, apiKey);
|
||||
if (!resp.ok) {
|
||||
return { output: `Job not found (${resp.status})`, isError: true };
|
||||
}
|
||||
|
||||
const data = await resp.json() as any;
|
||||
const p = data.progress || {};
|
||||
const statusLine = data.status === 'completed'
|
||||
? `完了 (${data.document_name})`
|
||||
: data.status === 'failed'
|
||||
? `失敗: ${data.error || 'unknown'}`
|
||||
: `処理中: VLM ${p.vlm_completed || 0}/${p.total_pages || 0}ページ, ツリー構築: ${p.tree_built ? '完了' : '未完了'}`;
|
||||
|
||||
return { output: `ジョブ ${data.job_id}: ${statusLine}`, isError: false };
|
||||
} catch (e: any) {
|
||||
if (e.name === 'AbortError') {
|
||||
return { output: `IngestStatus timeout: DKS server did not respond within 10s`, isError: true };
|
||||
}
|
||||
return { output: `IngestStatus error: ${e.message}`, isError: true };
|
||||
}
|
||||
}
|
||||
|
||||
async function executeSearchKnowledge(input: Record<string, unknown>, ctx: ToolContext): Promise<ToolResult> {
|
||||
const serviceUrl = getServiceUrl(ctx);
|
||||
if (!serviceUrl) return { output: 'Knowledge service not configured', isError: true };
|
||||
|
||||
const namespace = input.namespace as string;
|
||||
const query = input.query as string;
|
||||
const apiKey = getApiKey(ctx, namespace);
|
||||
if (!apiKey) return { output: `Namespace "${namespace}" not configured`, isError: true };
|
||||
|
||||
const startMs = Date.now();
|
||||
try {
|
||||
const resp = await dksFetch(serviceUrl, `/namespaces/${namespace}/search`, apiKey, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ query }),
|
||||
timeoutMs: 30000,
|
||||
});
|
||||
|
||||
if (!resp.ok) {
|
||||
const errText = await resp.text();
|
||||
const output = `Search failed (${resp.status}): ${errText}`;
|
||||
appendKnowledgeHistory(ctx, {
|
||||
timestamp: new Date().toISOString(), tool: 'SearchKnowledge',
|
||||
namespace, query, error: output, durationMs: Date.now() - startMs,
|
||||
});
|
||||
return { output, isError: true };
|
||||
}
|
||||
|
||||
const data = await resp.json() as any;
|
||||
|
||||
// DKS 生レスポンスを logs/raw/ に保存
|
||||
saveRawResponse(ctx, 'SearchKnowledge', { query, namespace, response: data });
|
||||
|
||||
const sections = data.sections || [];
|
||||
const rawImageUrls: string[] = data.page_image_urls || [];
|
||||
|
||||
// Download page images to workspace so ReadImage can access them
|
||||
const { localPaths, errors: imgErrors } = await downloadPageImages(
|
||||
serviceUrl, apiKey, rawImageUrls, ctx, namespace,
|
||||
);
|
||||
|
||||
// Format response
|
||||
const lines: string[] = [];
|
||||
for (const section of sections) {
|
||||
lines.push(`## ${section.title} (${section.doc_name}, pages: ${section.pages.join(', ')})`);
|
||||
lines.push(section.content);
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
if (localPaths.length > 0) {
|
||||
lines.push('### ページ画像（ReadImage で閲覧可能）');
|
||||
for (const p of localPaths) {
|
||||
lines.push(`- ${p}`);
|
||||
}
|
||||
}
|
||||
if (imgErrors.length > 0) {
|
||||
lines.push(`\n[画像ダウンロードエラー: ${imgErrors.length}件]`);
|
||||
}
|
||||
|
||||
if (data.system_status?.message) {
|
||||
lines.push(`\n[Info] ${data.system_status.message}`);
|
||||
}
|
||||
|
||||
const output = lines.join('\n');
|
||||
|
||||
appendKnowledgeHistory(ctx, {
|
||||
timestamp: new Date().toISOString(), tool: 'SearchKnowledge',
|
||||
namespace, query, sectionsFound: sections.length,
|
||||
imagesDownloaded: localPaths.length, imageErrors: imgErrors.length,
|
||||
durationMs: Date.now() - startMs,
|
||||
});
|
||||
|
||||
return { output, isError: false };
|
||||
} catch (e: any) {
|
||||
const error = e.name === 'AbortError'
|
||||
? `SearchKnowledge timeout: DKS server did not respond within 30s`
|
||||
: `SearchKnowledge error: ${e.message}`;
|
||||
appendKnowledgeHistory(ctx, {
|
||||
timestamp: new Date().toISOString(), tool: 'SearchKnowledge',
|
||||
namespace, query, error, durationMs: Date.now() - startMs,
|
||||
});
|
||||
return { output: error, isError: true };
|
||||
}
|
||||
}
|
||||
|
||||
async function executeListNamespaces(ctx: ToolContext): Promise<ToolResult> {
|
||||
const serviceUrl = getServiceUrl(ctx);
|
||||
const namespaces = getNamespaces(ctx);
|
||||
|
||||
if (!serviceUrl || !namespaces) {
|
||||
return { output: 'Knowledge service not configured', isError: true };
|
||||
}
|
||||
|
||||
const names = Object.keys(namespaces);
|
||||
if (names.length === 0) {
|
||||
return { output: '利用可能なネームスペースはありません', isError: false };
|
||||
}
|
||||
|
||||
appendKnowledgeHistory(ctx, {
|
||||
timestamp: new Date().toISOString(), tool: 'ListNamespaces',
|
||||
namespacesFound: names.length,
|
||||
});
|
||||
|
||||
return { output: `利用可能なネームスペース:\n${names.map(n => `- ${n}`).join('\n')}`, isError: false };
|
||||
}
|
||||
|
||||
async function executeListDocuments(input: Record<string, unknown>, ctx: ToolContext): Promise<ToolResult> {
|
||||
const serviceUrl = getServiceUrl(ctx);
|
||||
if (!serviceUrl) return { output: 'Knowledge service not configured', isError: true };
|
||||
|
||||
const namespace = input.namespace as string;
|
||||
const apiKey = getApiKey(ctx, namespace);
|
||||
if (!apiKey) return { output: `Namespace "${namespace}" not configured`, isError: true };
|
||||
|
||||
const startMs = Date.now();
|
||||
try {
|
||||
const resp = await dksFetch(serviceUrl, `/namespaces/${namespace}/documents`, apiKey);
|
||||
if (!resp.ok) {
|
||||
return { output: `ListDocuments failed (${resp.status})`, isError: true };
|
||||
}
|
||||
|
||||
const data = await resp.json() as any;
|
||||
const docs = data.documents || [];
|
||||
if (docs.length === 0) {
|
||||
return { output: `"${namespace}" にはまだ文書がありません`, isError: false };
|
||||
}
|
||||
|
||||
const lines = docs.map((d: any) => `- ${d.name} (${d.page_count}ページ, id: ${d.id})`);
|
||||
const output = `"${namespace}" の文書一覧:\n${lines.join('\n')}`;
|
||||
|
||||
appendKnowledgeHistory(ctx, {
|
||||
timestamp: new Date().toISOString(), tool: 'ListDocuments',
|
||||
namespace, documentsFound: docs.length, durationMs: Date.now() - startMs,
|
||||
});
|
||||
|
||||
return { output, isError: false };
|
||||
} catch (e: any) {
|
||||
const error = e.name === 'AbortError'
|
||||
? `ListDocuments timeout: DKS server did not respond within 10s`
|
||||
: `ListDocuments error: ${e.message}`;
|
||||
appendKnowledgeHistory(ctx, {
|
||||
timestamp: new Date().toISOString(), tool: 'ListDocuments',
|
||||
namespace, error, durationMs: Date.now() - startMs,
|
||||
});
|
||||
return { output: error, isError: true };
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user