185 lines
15 KiB
JavaScript
185 lines
15 KiB
JavaScript
// Local-only integration check: actual browser audio processing and AAO relay, synthetic ASR and translation servers.
|
|
import { dirname, resolve } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { createRequire } from 'node:module';
|
|
import { createServer } from 'node:http';
|
|
import { mkdtempSync, mkdirSync, readFileSync, readdirSync, rmSync } from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import { once } from 'node:events';
|
|
import assert from 'node:assert/strict';
|
|
const root=resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
|
const require=createRequire(root+'/package.json');
|
|
const express=require('express');
|
|
const Database=require('better-sqlite3');
|
|
const {WebSocketServer}=require('ws');
|
|
const {chromium}=require('playwright');
|
|
const {createTranscriptionApi}=await import(root+'/src/bridge/transcription-api.ts');
|
|
const {attachTranscriptionWs}=await import(root+'/src/bridge/transcription-ws.ts');
|
|
const upstream=new WebSocketServer({port:0}); await once(upstream,'listening');
|
|
let audioFrames=0, finals=0;
|
|
upstream.on('connection', ws=>{
|
|
let n=0;
|
|
ws.send('{"type":"ready"}');
|
|
ws.on('message',(data,binary)=>{
|
|
if(binary){audioFrames++;n++; if(n%3===0) ws.send(JSON.stringify({type:'transcript',text:`議題は AAO の文字起こし機能です。途中結果 ${n}`,is_final:false,segment_id:'session',segments:[]}));}
|
|
else {const m=JSON.parse(data.toString());if(m.type==='config')ws.send('{"type":"config_ack"}');if(m.type==='end'){finals++;ws.send(JSON.stringify({type:'transcript',text:'議題は AAO の文字起こし機能です。次回までに動作を確認します。',is_final:true,segment_id:'session',segments:[{id:'s1',start:0,end:2,speaker:'1',text:'議題は AAO の文字起こし機能です。',timestamp_source:'model'}]}));}}
|
|
});
|
|
});
|
|
const translatedRequests=[];
|
|
const translationApp=express();translationApp.use(express.json());
|
|
translationApp.post('/v1/chat/completions',(req,res)=>{
|
|
const content=req.body.messages?.[0]?.content;
|
|
assert.equal(req.body.model,'test-translategemma');
|
|
assert.equal(typeof content,'string');
|
|
assert(content.startsWith('<<<source>>>ja<<<target>>>en<<<text>>>'));
|
|
const original=content.split('<<<text>>>')[1]; translatedRequests.push(original);
|
|
setTimeout(()=>res.json({choices:[{finish_reason:'stop',message:{content:`Translated: ${original}`}}]}),100);
|
|
});
|
|
const translationServer=translationApp.listen(0,'127.0.0.1');await once(translationServer,'listening');
|
|
const translationEndpoint=`http://127.0.0.1:${translationServer.address().port}/v1`;
|
|
const tools={speechStreamingUrl:`ws://127.0.0.1:${upstream.address().port}/v1/audio/stream`,speechStreamingTimeout:10};
|
|
const configuredEndpoint=tools.speechStreamingUrl; tools.speechStreamingUrl='';
|
|
const db=new Database(':memory:');
|
|
const fixtureRoot=mkdtempSync(resolve(tmpdir(),'aao-chat-transcription-'));
|
|
const timestamp=new Date().toISOString();
|
|
const space={id:'browser-space',kind:'personal',title:'Browser Workspace',description:'',ownerId:'browser-test',visibility:'private',visibilityScopeOrgId:null,status:'open',brandColor:null,workspaceDir:fixtureRoot,createdAt:timestamp,updatedAt:timestamp,myRole:'owner'};
|
|
const tasks=[1,2].map(id=>({id,title:id===1?'会議 A':'会議 B',body:'Browser transcription check',pieceName:'chat',profile:'default',outputFormat:'text',askPolicy:'auto',priority:'normal',state:'succeeded',workspacePath:resolve(fixtureRoot,`task-${id}`),ownerId:'browser-test',ownerName:'Browser Test',visibility:'private',spaceId:space.id,createdAt:timestamp,updatedAt:timestamp,latestJob:null,subtasks:[],tags:[],allowedTools:[]}));
|
|
for(const task of tasks) mkdirSync(task.workspacePath,{recursive:true});
|
|
const provenance=[];
|
|
const repo={getLatestJobForIssue:async()=>null,getLocalTask:async id=>tasks.find(task=>task.id===id)??null,getSpace:()=>space,getSpaceMemberRole:()=> 'owner',userCanViewSpace:()=>true,recordProvenance:entry=>provenance.push(entry)};
|
|
|
|
const app=express();
|
|
app.use((req,res,next)=>{req.user={id:'browser-test',role:'admin',email:'[email protected]',name:'Browser Test'};next();});
|
|
app.use('/api/transcription',createTranscriptionApi({db,repo,authActive:true,getTools:()=>tools}));
|
|
app.use(express.json());
|
|
app.get('/api/auth/me',(req,res)=>res.json(req.user));
|
|
app.get('/api/setup/status',(_req,res)=>res.json({needsSetup:false,authActive:true}));
|
|
app.get('/api/config',(_req,res)=>res.json({config:{configVersion:2,tools,storage:{},llm:{workers:[]}},overriddenByEnv:{}}));
|
|
app.put('/api/config',(req,res)=>{Object.assign(tools,req.body.tools);res.json({ok:true});});
|
|
app.get('/api/local/spaces',(_req,res)=>res.json([space]));
|
|
app.get('/api/local/spaces/:id',(_req,res)=>res.json(space));
|
|
app.get('/api/local/spaces/:id/members',(_req,res)=>res.json([{userId:'browser-test',name:'Browser Test',role:'owner',isOwner:true}]));
|
|
app.get('/api/local/tasks',(_req,res)=>res.json({tasks}));
|
|
app.get('/api/local/tasks/:id', (req,res)=>res.json({task:tasks.find(task=>task.id===Number(req.params.id))}));
|
|
app.get('/api/local/tasks/:id/overview-metrics',(_req,res)=>res.json({metrics:{collecting:false,toolCalls:null,compactions:null,activeDurationMs:null,workerId:null,workerName:null,model:null,decodeTps:null,decodeTpsMeasuredAt:null,jobId:null,jobStatus:null}}));
|
|
app.get('/api/local/tasks/:id/comments',(_req,res)=>res.json({comments:[]}));
|
|
app.get('/api/local/tasks/:id/movement-history',(_req,res)=>res.json({events:[]}));
|
|
app.get('/api/local/tasks/:id/executions',(_req,res)=>res.json({executions:[],active:false}));
|
|
app.get('/api/local/tasks/:id/files',(_req,res)=>res.json({basePath:'',path:'',entries:[],truncated:false}));
|
|
app.get('/api/local/spaces/:id/files',(_req,res)=>res.json({basePath:'',path:'',entries:[],truncated:false}));
|
|
app.get('/api/local/tasks/:id/console/status',(_req,res)=>res.json({active:false}));
|
|
app.get('/api/local/browser/sessions/task-session/:id',(_req,res)=>res.json({available:false}));
|
|
|
|
app.get('/api/pieces',(_req,res)=>res.json({pieces:[]}));
|
|
app.get('/api/pieces/:name',(req,res)=>res.json({piece:{name:req.params.name,description:'',movements:[],max_movements:1,initial_movement:''},source:'builtin'}));
|
|
app.get('/api/users/me/pets',(_req,res)=>res.json({pets:[],settings:{enabled:false,activePetId:null,size:48,position:'bottom-right',sound:false,reducedMotion:true,toolSparkEnabled:false,workerPets:{}}}));
|
|
app.get('/api/notifications/capabilities',(_req,res)=>res.json({}));
|
|
app.use('/api',(_req,res)=>res.status(404).json({error:'Not included in the local browser fixture'}));
|
|
app.use('/ui',express.static(root+'/ui/dist'));
|
|
app.get('/ui/*',(_req,res)=>res.sendFile(root+'/ui/dist/index.html'));
|
|
const server=createServer(app);
|
|
const stop=attachTranscriptionWs(server,{authActive:true,authenticateUpgrade:async()=>({id:'browser-test',role:'admin'}),getTools:()=>tools});
|
|
server.listen(0,'127.0.0.1');await once(server,'listening');
|
|
const base=`http://127.0.0.1:${server.address().port}`;
|
|
let browser, page;
|
|
const errors=[];
|
|
try{
|
|
browser=await chromium.launch({headless:true,args:['--no-sandbox','--use-fake-device-for-media-stream','--use-fake-ui-for-media-stream','--autoplay-policy=no-user-gesture-required']});
|
|
const context=await browser.newContext({viewport:{width:1440,height:1000},permissions:['microphone']});
|
|
await context.addInitScript(()=>{
|
|
localStorage.setItem('maestro.lang','ja');
|
|
navigator.mediaDevices.getDisplayMedia=async()=>{
|
|
const ctx=new AudioContext(); const oscillator=ctx.createOscillator(); const output=ctx.createMediaStreamDestination();oscillator.connect(output);oscillator.start();
|
|
window.__desktopAudioContext=ctx; return output.stream;
|
|
};
|
|
});
|
|
page=await context.newPage(); page.on('pageerror',e=>errors.push(e.message));
|
|
await page.goto(base+'/ui/?page=transcription');
|
|
await page.getByText(/文字起こしはチャット右側/).waitFor();
|
|
await page.goto(base+'/ui/?space=browser-space&chat=1&tab=transcription');
|
|
const dockA=page.locator('[data-transcription-task="1"]');
|
|
await dockA.getByRole('heading',{name:/^(文字起こし|Transcription)$/}).waitFor();
|
|
await page.getByRole('button',{name:'設定を開く',exact:true}).click();
|
|
await page.getByPlaceholder('ws://gpu-server:8000/v1/audio/stream').fill(configuredEndpoint);
|
|
await page.locator('#translation-server-url').fill(translationEndpoint);
|
|
await page.locator('#translation-model').fill('test-translategemma');
|
|
const savedConfig=page.waitForResponse(r=>r.url().endsWith('/api/config') && r.request().method()==='PUT');
|
|
await page.getByRole('button',{name:/保存して反映|Save and apply/i}).click();
|
|
assert.equal((await savedConfig).status(),200);
|
|
await page.getByRole('button',{name:/^(ワークスペース|Workspaces)$/}).first().click();
|
|
if(!await dockA.isVisible()) await page.getByTestId('space-chat-tab-transcription').click();
|
|
await dockA.getByRole('heading',{name:/^(文字起こし|Transcription)$/}).waitFor();
|
|
await page.getByLabel('原文言語').selectOption('ja');
|
|
await page.getByLabel('翻訳先言語').selectOption('en');
|
|
await page.getByRole('checkbox',{name:'逐次翻訳',exact:true}).check();
|
|
await page.getByRole('button',{name:'開始',exact:true}).click();
|
|
await page.getByText('文字起こし中',{exact:true}).waitFor();
|
|
await page.waitForFunction(()=>[...document.querySelectorAll('textarea')].some(x=>x.value.includes('途中結果')));
|
|
await page.getByText('Translated: 議題は AAO の文字起こし機能です。',{exact:true}).first().waitFor();
|
|
const before=audioFrames;
|
|
// Navigate away via actual App navigation, then return; capture stays mounted.
|
|
await page.screenshot({path:'/tmp/aao-transcription-live-check.png',fullPage:true});
|
|
await page.getByRole('button',{name:/^(ヘルプ|Help)$/}).first().click();
|
|
await page.waitForTimeout(800);
|
|
assert(audioFrames>before,'audio continues on another tab');
|
|
await page.getByRole('button',{name:'録音中のチャットを開く',exact:true}).click();
|
|
await dockA.getByText('文字起こし中',{exact:true}).waitFor();
|
|
// Change chat through the real workspace list: A must keep recording and B must not acquire audio.
|
|
const expand=page.getByTestId('space-chat-list-expand');
|
|
if(await expand.isVisible()) await expand.click();
|
|
await page.locator('[data-testid="space-chat-row"][data-task-id="2"]').getByText('会議 B',{exact:true}).click();
|
|
const dockB=page.locator('[data-transcription-task="2"]');
|
|
if(!await dockB.isVisible()) await page.getByTestId('space-chat-tab-transcription').click();
|
|
await dockB.getByRole('heading',{name:/^(文字起こし|Transcription)$/}).waitFor();
|
|
assert(await dockB.getByRole('button',{name:'開始',exact:true}).isDisabled(),'chat B cannot start while A records');
|
|
const beforeB=audioFrames;await page.waitForTimeout(700);assert(audioFrames>beforeB,'A audio continues while B is visible');
|
|
const firstSave=page.waitForResponse(r=>r.url().endsWith('/api/transcription/recordings') && r.request().method()==='POST');
|
|
await page.getByRole('button',{name:'録音を停止して保存',exact:true}).click();
|
|
const firstStored=await (await firstSave).json();
|
|
assert.equal(firstStored.recording.taskId,1);
|
|
assert(firstStored.workspaceFiles && !firstStored.workspaceError,'original transcript is also written into A workspace');
|
|
assert.equal(finals,2); assert(audioFrames>0);
|
|
assert(await dockB.isVisible(),'global stop keeps chat B displayed');
|
|
await page.getByRole('button',{name:'録音を停止して保存',exact:true}).waitFor({state:'hidden'});
|
|
assert(await dockB.getByRole('button',{name:'開始',exact:true}).isEnabled(),'finishing hidden A releases the recording lock');
|
|
await page.locator('[data-testid="space-chat-row"][data-task-id="1"]').getByText('会議 A',{exact:true}).click();
|
|
if(!await dockA.isVisible()) await page.getByTestId('space-chat-tab-transcription').click();
|
|
await dockA.getByRole('heading',{name:/^(文字起こし|Transcription)$/}).waitFor();
|
|
assert.equal(await page.locator('.transcription-transcript article').count(),2);
|
|
await page.waitForFunction(()=>{
|
|
const pairs=[...document.querySelectorAll('.transcription-pair')];
|
|
return pairs.length===4 && pairs.every(p=>p.textContent.includes('翻訳済み'));
|
|
});
|
|
await page.getByText('未保存の変更があります。',{exact:true}).waitFor();
|
|
const savedTranslation=page.waitForResponse(r=>r.url().endsWith('/api/transcription/recordings') && r.request().method()==='POST');
|
|
await page.getByRole('button',{name:'保存 / 再試行',exact:true}).click();
|
|
const saveResult=await (await savedTranslation).json();const stored=saveResult.recording;
|
|
assert.equal(stored.taskId,1);assert(!saveResult.workspaceError);
|
|
const taskARecordings=await (await context.request.get(base+'/api/transcription/recordings?taskId=1')).json();
|
|
const taskBRecordings=await (await context.request.get(base+'/api/transcription/recordings?taskId=2')).json();
|
|
assert.equal(taskARecordings.recordings.length,1);assert.equal(taskBRecordings.recordings.length,0);
|
|
const fileJson=JSON.parse(readFileSync(resolve(tasks[0].workspacePath,saveResult.workspaceFiles.json),'utf8'));
|
|
assert.deepEqual(fileJson.entries,stored.entries);assert.equal(fileJson.taskId,1);
|
|
const fileText=readFileSync(resolve(tasks[0].workspacePath,saveResult.workspaceFiles.txt),'utf8');
|
|
assert(fileText.includes('Translated:') && fileText.includes('原文') && fileText.includes('改訂'));
|
|
assert.equal(readdirSync(tasks[1].workspacePath).length,0,'B workspace remains untouched');
|
|
assert(provenance.some(entry=>entry.taskId===1 && entry.relPath.endsWith('.json')));
|
|
assert(stored.entries.every(e=>e.blocks.every(b=>b.translation?.sourceRevision===b.revision && b.translation.sourceText===b.text)));
|
|
assert.equal(new Set(stored.entries.flatMap(e=>e.blocks.map(b=>b.id))).size,4);
|
|
await page.screenshot({path:'/tmp/aao-transcription-wide-check.png',fullPage:true});
|
|
await page.reload();
|
|
await page.locator('[data-transcription-task="1"] .transcription-recording button').first().click();
|
|
await page.locator('.transcription-transcript article').nth(1).waitFor();
|
|
assert.equal(await page.locator('.transcription-transcript article').count(),2);
|
|
assert.equal(await page.getByRole('checkbox',{name:'逐次翻訳',exact:true}).isChecked(),false);
|
|
assert.equal(await page.locator('.transcription-pair').count(),4);
|
|
assert.equal(await page.getByText('Translated: 次回までに動作を確認します。',{exact:true}).count(),2);
|
|
const downloaded=page.waitForEvent('download');await page.getByRole('button',{name:'TXT',exact:true}).click();await downloaded;
|
|
await page.setViewportSize({width:390,height:844});
|
|
await page.screenshot({path:'/tmp/aao-transcription-narrow-check.png',fullPage:true});
|
|
assert(await page.evaluate(()=>document.documentElement.scrollWidth<=innerWidth),'no horizontal overflow');
|
|
assert(await page.getByTestId('space-chat-tab-transcription').isVisible(),'transcription tab remains reachable on mobile');
|
|
console.log(JSON.stringify({audioFrames,finals,entries:2,translatedRequests:translatedRequests.length,taskId:1,workspaceFiles:saveResult.workspaceFiles,errors,screenshots:['/tmp/aao-transcription-wide-check.png','/tmp/aao-transcription-narrow-check.png']}));
|
|
assert.equal(errors.length,0);
|
|
}catch(error){console.error(JSON.stringify({url:page?.url(),errors,body:await page?.locator('body').innerText().catch(()=> '')}));await page?.screenshot({path:'/tmp/aao-transcription-failure.png',fullPage:true}).catch(()=>{});throw error;}finally{await browser?.close();stop();server.closeAllConnections();server.close();upstream.clients.forEach(s=>s.terminate());upstream.close();translationServer.closeAllConnections();translationServer.close();db.close();rmSync(fixtureRoot,{recursive:true,force:true});}
|