// Local-only integration check: actual browser audio processing and AAO relay, synthetic ASR and translation servers. import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { createRequire } from 'node:module'; import { createServer } from 'node:http'; import { mkdtempSync, mkdirSync, readFileSync, readdirSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { once } from 'node:events'; import assert from 'node:assert/strict'; const root=resolve(dirname(fileURLToPath(import.meta.url)), '..'); const require=createRequire(root+'/package.json'); const express=require('express'); const Database=require('better-sqlite3'); const {WebSocketServer}=require('ws'); const {chromium}=require('playwright'); const {createTranscriptionApi}=await import(root+'/src/bridge/transcription-api.ts'); const {attachTranscriptionWs}=await import(root+'/src/bridge/transcription-ws.ts'); const upstream=new WebSocketServer({port:0}); await once(upstream,'listening'); let audioFrames=0, finals=0; upstream.on('connection', ws=>{ let n=0; ws.send('{"type":"ready"}'); ws.on('message',(data,binary)=>{ if(binary){audioFrames++;n++; if(n%3===0) ws.send(JSON.stringify({type:'transcript',text:`議題は AAO の文字起こし機能です。途中結果 ${n}`,is_final:false,segment_id:'session',segments:[]}));} else {const m=JSON.parse(data.toString());if(m.type==='config')ws.send('{"type":"config_ack"}');if(m.type==='end'){finals++;ws.send(JSON.stringify({type:'transcript',text:'議題は AAO の文字起こし機能です。次回までに動作を確認します。',is_final:true,segment_id:'session',segments:[{id:'s1',start:0,end:2,speaker:'1',text:'議題は AAO の文字起こし機能です。',timestamp_source:'model'}]}));}} }); }); const translatedRequests=[]; const translationApp=express();translationApp.use(express.json()); translationApp.post('/v1/chat/completions',(req,res)=>{ const content=req.body.messages?.[0]?.content; assert.equal(req.body.model,'test-translategemma'); assert.equal(typeof content,'string'); assert(content.startsWith('<<>>ja<<>>en<<>>')); const original=content.split('<<>>')[1]; translatedRequests.push(original); setTimeout(()=>res.json({choices:[{finish_reason:'stop',message:{content:`Translated: ${original}`}}]}),100); }); const translationServer=translationApp.listen(0,'127.0.0.1');await once(translationServer,'listening'); const translationEndpoint=`http://127.0.0.1:${translationServer.address().port}/v1`; const tools={speechStreamingUrl:`ws://127.0.0.1:${upstream.address().port}/v1/audio/stream`,speechStreamingTimeout:10}; const configuredEndpoint=tools.speechStreamingUrl; tools.speechStreamingUrl=''; const db=new Database(':memory:'); const fixtureRoot=mkdtempSync(resolve(tmpdir(),'aao-chat-transcription-')); const timestamp=new Date().toISOString(); const space={id:'browser-space',kind:'personal',title:'Browser Workspace',description:'',ownerId:'browser-test',visibility:'private',visibilityScopeOrgId:null,status:'open',brandColor:null,workspaceDir:fixtureRoot,createdAt:timestamp,updatedAt:timestamp,myRole:'owner'}; const tasks=[1,2].map(id=>({id,title:id===1?'会議 A':'会議 B',body:'Browser transcription check',pieceName:'chat',profile:'default',outputFormat:'text',askPolicy:'auto',priority:'normal',state:'succeeded',workspacePath:resolve(fixtureRoot,`task-${id}`),ownerId:'browser-test',ownerName:'Browser Test',visibility:'private',spaceId:space.id,createdAt:timestamp,updatedAt:timestamp,latestJob:null,subtasks:[],tags:[],allowedTools:[]})); for(const task of tasks) mkdirSync(task.workspacePath,{recursive:true}); const provenance=[]; const repo={getLatestJobForIssue:async()=>null,getLocalTask:async id=>tasks.find(task=>task.id===id)??null,getSpace:()=>space,getSpaceMemberRole:()=> 'owner',userCanViewSpace:()=>true,recordProvenance:entry=>provenance.push(entry)}; const app=express(); app.use((req,res,next)=>{req.user={id:'browser-test',role:'admin',email:'test@example.invalid',name:'Browser Test'};next();}); app.use('/api/transcription',createTranscriptionApi({db,repo,authActive:true,getTools:()=>tools})); app.use(express.json()); app.get('/api/auth/me',(req,res)=>res.json(req.user)); app.get('/api/setup/status',(_req,res)=>res.json({needsSetup:false,authActive:true})); app.get('/api/config',(_req,res)=>res.json({config:{configVersion:2,tools,storage:{},llm:{workers:[]}},overriddenByEnv:{}})); app.put('/api/config',(req,res)=>{Object.assign(tools,req.body.tools);res.json({ok:true});}); app.get('/api/local/spaces',(_req,res)=>res.json([space])); app.get('/api/local/spaces/:id',(_req,res)=>res.json(space)); app.get('/api/local/spaces/:id/members',(_req,res)=>res.json([{userId:'browser-test',name:'Browser Test',role:'owner',isOwner:true}])); app.get('/api/local/tasks',(_req,res)=>res.json({tasks})); app.get('/api/local/tasks/:id', (req,res)=>res.json({task:tasks.find(task=>task.id===Number(req.params.id))})); app.get('/api/local/tasks/:id/overview-metrics',(_req,res)=>res.json({metrics:{collecting:false,toolCalls:null,compactions:null,activeDurationMs:null,workerId:null,workerName:null,model:null,decodeTps:null,decodeTpsMeasuredAt:null,jobId:null,jobStatus:null}})); app.get('/api/local/tasks/:id/comments',(_req,res)=>res.json({comments:[]})); app.get('/api/local/tasks/:id/movement-history',(_req,res)=>res.json({events:[]})); app.get('/api/local/tasks/:id/executions',(_req,res)=>res.json({executions:[],active:false})); app.get('/api/local/tasks/:id/files',(_req,res)=>res.json({basePath:'',path:'',entries:[],truncated:false})); app.get('/api/local/spaces/:id/files',(_req,res)=>res.json({basePath:'',path:'',entries:[],truncated:false})); app.get('/api/local/tasks/:id/console/status',(_req,res)=>res.json({active:false})); app.get('/api/local/browser/sessions/task-session/:id',(_req,res)=>res.json({available:false})); app.get('/api/pieces',(_req,res)=>res.json({pieces:[]})); app.get('/api/pieces/:name',(req,res)=>res.json({piece:{name:req.params.name,description:'',movements:[],max_movements:1,initial_movement:''},source:'builtin'})); app.get('/api/users/me/pets',(_req,res)=>res.json({pets:[],settings:{enabled:false,activePetId:null,size:48,position:'bottom-right',sound:false,reducedMotion:true,toolSparkEnabled:false,workerPets:{}}})); app.get('/api/notifications/capabilities',(_req,res)=>res.json({})); app.use('/api',(_req,res)=>res.status(404).json({error:'Not included in the local browser fixture'})); app.use('/ui',express.static(root+'/ui/dist')); app.get('/ui/*',(_req,res)=>res.sendFile(root+'/ui/dist/index.html')); const server=createServer(app); const stop=attachTranscriptionWs(server,{authActive:true,authenticateUpgrade:async()=>({id:'browser-test',role:'admin'}),getTools:()=>tools}); server.listen(0,'127.0.0.1');await once(server,'listening'); const base=`http://127.0.0.1:${server.address().port}`; let browser, page; const errors=[]; try{ browser=await chromium.launch({headless:true,args:['--no-sandbox','--use-fake-device-for-media-stream','--use-fake-ui-for-media-stream','--autoplay-policy=no-user-gesture-required']}); const context=await browser.newContext({viewport:{width:1440,height:1000},permissions:['microphone']}); await context.addInitScript(()=>{ localStorage.setItem('maestro.lang','ja'); navigator.mediaDevices.getDisplayMedia=async()=>{ const ctx=new AudioContext(); const oscillator=ctx.createOscillator(); const output=ctx.createMediaStreamDestination();oscillator.connect(output);oscillator.start(); window.__desktopAudioContext=ctx; return output.stream; }; }); page=await context.newPage(); page.on('pageerror',e=>errors.push(e.message)); await page.goto(base+'/ui/?page=transcription'); await page.getByText(/文字起こしはチャット右側/).waitFor(); await page.goto(base+'/ui/?space=browser-space&chat=1&tab=transcription'); const dockA=page.locator('[data-transcription-task="1"]'); await dockA.getByRole('heading',{name:/^(文字起こし|Transcription)$/}).waitFor(); await page.getByRole('button',{name:'設定を開く',exact:true}).click(); await page.getByPlaceholder('ws://gpu-server:8000/v1/audio/stream').fill(configuredEndpoint); await page.locator('#translation-server-url').fill(translationEndpoint); await page.locator('#translation-model').fill('test-translategemma'); const savedConfig=page.waitForResponse(r=>r.url().endsWith('/api/config') && r.request().method()==='PUT'); await page.getByRole('button',{name:/保存して反映|Save and apply/i}).click(); assert.equal((await savedConfig).status(),200); await page.getByRole('button',{name:/^(ワークスペース|Workspaces)$/}).first().click(); if(!await dockA.isVisible()) await page.getByTestId('space-chat-tab-transcription').click(); await dockA.getByRole('heading',{name:/^(文字起こし|Transcription)$/}).waitFor(); await page.getByLabel('原文言語').selectOption('ja'); await page.getByLabel('翻訳先言語').selectOption('en'); await page.getByRole('checkbox',{name:'逐次翻訳',exact:true}).check(); await page.getByRole('button',{name:'開始',exact:true}).click(); await page.getByText('文字起こし中',{exact:true}).waitFor(); await page.waitForFunction(()=>[...document.querySelectorAll('textarea')].some(x=>x.value.includes('途中結果'))); await page.getByText('Translated: 議題は AAO の文字起こし機能です。',{exact:true}).first().waitFor(); const before=audioFrames; // Navigate away via actual App navigation, then return; capture stays mounted. await page.screenshot({path:'/tmp/aao-transcription-live-check.png',fullPage:true}); await page.getByRole('button',{name:/^(ヘルプ|Help)$/}).first().click(); await page.waitForTimeout(800); assert(audioFrames>before,'audio continues on another tab'); await page.getByRole('button',{name:'録音中のチャットを開く',exact:true}).click(); await dockA.getByText('文字起こし中',{exact:true}).waitFor(); // Change chat through the real workspace list: A must keep recording and B must not acquire audio. const expand=page.getByTestId('space-chat-list-expand'); if(await expand.isVisible()) await expand.click(); await page.locator('[data-testid="space-chat-row"][data-task-id="2"]').getByText('会議 B',{exact:true}).click(); const dockB=page.locator('[data-transcription-task="2"]'); if(!await dockB.isVisible()) await page.getByTestId('space-chat-tab-transcription').click(); await dockB.getByRole('heading',{name:/^(文字起こし|Transcription)$/}).waitFor(); assert(await dockB.getByRole('button',{name:'開始',exact:true}).isDisabled(),'chat B cannot start while A records'); const beforeB=audioFrames;await page.waitForTimeout(700);assert(audioFrames>beforeB,'A audio continues while B is visible'); const firstSave=page.waitForResponse(r=>r.url().endsWith('/api/transcription/recordings') && r.request().method()==='POST'); await page.getByRole('button',{name:'録音を停止して保存',exact:true}).click(); const firstStored=await (await firstSave).json(); assert.equal(firstStored.recording.taskId,1); assert(firstStored.workspaceFiles && !firstStored.workspaceError,'original transcript is also written into A workspace'); assert.equal(finals,2); assert(audioFrames>0); assert(await dockB.isVisible(),'global stop keeps chat B displayed'); await page.getByRole('button',{name:'録音を停止して保存',exact:true}).waitFor({state:'hidden'}); assert(await dockB.getByRole('button',{name:'開始',exact:true}).isEnabled(),'finishing hidden A releases the recording lock'); await page.locator('[data-testid="space-chat-row"][data-task-id="1"]').getByText('会議 A',{exact:true}).click(); if(!await dockA.isVisible()) await page.getByTestId('space-chat-tab-transcription').click(); await dockA.getByRole('heading',{name:/^(文字起こし|Transcription)$/}).waitFor(); assert.equal(await page.locator('.transcription-transcript article').count(),2); await page.waitForFunction(()=>{ const pairs=[...document.querySelectorAll('.transcription-pair')]; return pairs.length===4 && pairs.every(p=>p.textContent.includes('翻訳済み')); }); await page.getByText('未保存の変更があります。',{exact:true}).waitFor(); const savedTranslation=page.waitForResponse(r=>r.url().endsWith('/api/transcription/recordings') && r.request().method()==='POST'); await page.getByRole('button',{name:'保存 / 再試行',exact:true}).click(); const saveResult=await (await savedTranslation).json();const stored=saveResult.recording; assert.equal(stored.taskId,1);assert(!saveResult.workspaceError); const taskARecordings=await (await context.request.get(base+'/api/transcription/recordings?taskId=1')).json(); const taskBRecordings=await (await context.request.get(base+'/api/transcription/recordings?taskId=2')).json(); assert.equal(taskARecordings.recordings.length,1);assert.equal(taskBRecordings.recordings.length,0); const fileJson=JSON.parse(readFileSync(resolve(tasks[0].workspacePath,saveResult.workspaceFiles.json),'utf8')); assert.deepEqual(fileJson.entries,stored.entries);assert.equal(fileJson.taskId,1); const fileText=readFileSync(resolve(tasks[0].workspacePath,saveResult.workspaceFiles.txt),'utf8'); assert(fileText.includes('Translated:') && fileText.includes('原文') && fileText.includes('改訂')); assert.equal(readdirSync(tasks[1].workspacePath).length,0,'B workspace remains untouched'); assert(provenance.some(entry=>entry.taskId===1 && entry.relPath.endsWith('.json'))); assert(stored.entries.every(e=>e.blocks.every(b=>b.translation?.sourceRevision===b.revision && b.translation.sourceText===b.text))); assert.equal(new Set(stored.entries.flatMap(e=>e.blocks.map(b=>b.id))).size,4); await page.screenshot({path:'/tmp/aao-transcription-wide-check.png',fullPage:true}); await page.reload(); await page.locator('[data-transcription-task="1"] .transcription-recording button').first().click(); await page.locator('.transcription-transcript article').nth(1).waitFor(); assert.equal(await page.locator('.transcription-transcript article').count(),2); assert.equal(await page.getByRole('checkbox',{name:'逐次翻訳',exact:true}).isChecked(),false); assert.equal(await page.locator('.transcription-pair').count(),4); assert.equal(await page.getByText('Translated: 次回までに動作を確認します。',{exact:true}).count(),2); const downloaded=page.waitForEvent('download');await page.getByRole('button',{name:'TXT',exact:true}).click();await downloaded; await page.setViewportSize({width:390,height:844}); await page.screenshot({path:'/tmp/aao-transcription-narrow-check.png',fullPage:true}); assert(await page.evaluate(()=>document.documentElement.scrollWidth<=innerWidth),'no horizontal overflow'); assert(await page.getByTestId('space-chat-tab-transcription').isVisible(),'transcription tab remains reachable on mobile'); console.log(JSON.stringify({audioFrames,finals,entries:2,translatedRequests:translatedRequests.length,taskId:1,workspaceFiles:saveResult.workspaceFiles,errors,screenshots:['/tmp/aao-transcription-wide-check.png','/tmp/aao-transcription-narrow-check.png']})); assert.equal(errors.length,0); }catch(error){console.error(JSON.stringify({url:page?.url(),errors,body:await page?.locator('body').innerText().catch(()=> '')}));await page?.screenshot({path:'/tmp/aao-transcription-failure.png',fullPage:true}).catch(()=>{});throw error;}finally{await browser?.close();stop();server.closeAllConnections();server.close();upstream.clients.forEach(s=>s.terminate());upstream.close();translationServer.closeAllConnections();translationServer.close();db.close();rmSync(fixtureRoot,{recursive:true,force:true});}