maestro/src/engine/tools/text-decode.test.ts

import { describe, expect, it } from 'vitest';
import iconv from 'iconv-lite';
import { detectTextEncoding, decodeToUtf8, encodeFromUtf8, decodeFileText, encodeFileText } from './text-decode.js';

const JP = 'こんにちは世界、これはテストです。\n日本語のテキスト。\n';

describe('detectTextEncoding', () => {
  it('keeps plain ASCII as utf-8', () => {
    const v = detectTextEncoding(Buffer.from('hello world\nsecond line\n', 'ascii'));
    expect(v).toEqual({ binary: false, encoding: 'utf-8' });
  });

  it('keeps valid UTF-8 Japanese as utf-8', () => {
    const v = detectTextEncoding(Buffer.from(JP, 'utf-8'));
    expect(v).toEqual({ binary: false, encoding: 'utf-8' });
  });

  it('recovers Shift_JIS (CP932) Japanese text as text, not binary', () => {
    const v = detectTextEncoding(iconv.encode(JP, 'Shift_JIS'));
    expect(v.binary).toBe(false);
    if (!v.binary) expect(v.encoding).toBe('shift_jis');
  });

  it('recovers EUC-JP Japanese text as text, not binary', () => {
    const v = detectTextEncoding(iconv.encode(JP, 'EUC-JP'));
    expect(v.binary).toBe(false);
    if (!v.binary) expect(v.encoding).toBe('euc-jp');
  });

  it('distinguishes kana-heavy EUC-JP from Shift_JIS (tie-break, codex regression)', () => {
    // Both decode without mojibake, but EUC-JP kana read as Shift_JIS lands in
    // halfwidth katakana; the body-text tie-break must pick the right one.
    const kana = 'あいうえお、こんにちは。さようなら\n';
    const euc = detectTextEncoding(iconv.encode(kana, 'EUC-JP'));
    expect(euc.binary).toBe(false);
    if (!euc.binary) expect(euc.encoding).toBe('euc-jp');
    const sjis = detectTextEncoding(iconv.encode(kana, 'Shift_JIS'));
    expect(sjis.binary).toBe(false);
    if (!sjis.binary) expect(sjis.encoding).toBe('shift_jis');
  });

  it('recovers windows-1252 / Latin-1 accented text as text', () => {
    const v = detectTextEncoding(iconv.encode('café résumé naïve Zürich\n', 'windows-1252'));
    expect(v.binary).toBe(false);
  });

  it('still reports a PNG as binary (magic byte)', () => {
    const png = Buffer.concat([
      Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
      Buffer.from('IHDR'),
      Buffer.from([0, 0, 0, 1, 0, 0, 0, 1, 8, 6, 0, 0, 0]),
    ]);
    const v = detectTextEncoding(png);
    expect(v.binary).toBe(true);
  });

  it('still reports NUL-heavy binary as binary', () => {
    const v = detectTextEncoding(Buffer.from([0x00, 0x01, 0x02, 0x00, 0x03, 0x00, 0xff, 0x00]));
    expect(v.binary).toBe(true);
  });

  it('rejects high-entropy binary that windows-1252 would decode without replacement chars', () => {
    // Regression (codex P2): bytes 0xA0–0xFF have no NUL/magic/C0 controls and
    // decode cleanly under cp1252, but are a symbol soup — not real text.
    const blob = Buffer.from(Array.from({ length: 100 }, (_, i) => 0xa0 + ((i * 37) % 96)));
    const v = detectTextEncoding(blob);
    expect(v.binary).toBe(true);
  });

  it('reports high-control random bytes (not valid UTF-8) as binary', () => {
    // 0x81/0x8f... with lots of C0 controls: not utf-8, high control ratio → binary.
    const junk = Buffer.from([
      0x81, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x8f, 0x90, 0x9d,
    ]);
    const v = detectTextEncoding(junk);
    expect(v.binary).toBe(true);
  });
});

describe('decodeToUtf8 / encodeFromUtf8 round-trip', () => {
  it('decodes Shift_JIS bytes to the original UTF-8 string', () => {
    const decoded = decodeToUtf8(iconv.encode(JP, 'Shift_JIS'), 'shift_jis');
    expect(decoded).toBe(JP);
  });

  it('re-encodes a UTF-8 edit back into Shift_JIS bytes losslessly', () => {
    const edited = JP.replace('テスト', 'サンプル');
    const bytes = encodeFromUtf8(edited, 'shift_jis');
    expect(iconv.decode(bytes, 'Shift_JIS')).toBe(edited);
    // and the bytes are genuinely Shift_JIS (not utf-8): decoding as utf-8 differs
    expect(bytes.toString('utf-8')).not.toBe(edited);
  });

  it('passes utf-8 through unchanged', () => {
    expect(decodeToUtf8(Buffer.from(JP, 'utf-8'), 'utf-8')).toBe(JP);
    expect(encodeFromUtf8(JP, 'utf-8').equals(Buffer.from(JP, 'utf-8'))).toBe(true);
  });
});

describe('decodeFileText (whole-file detection)', () => {
  it('detects Shift_JIS even when the first 8KB is pure ASCII', () => {
    // Long ASCII header (> SNIFF_HEAD_BYTES) then Shift_JIS Japanese tail.
    const header = Buffer.from('# '.padEnd(9000, 'header-comment ') + '\n', 'ascii');
    const tail = iconv.encode('本文は日本語です。\n', 'Shift_JIS');
    const full = Buffer.concat([header, tail]);
    const r = decodeFileText(full);
    expect(r.binary).toBe(false);
    if (!r.binary) {
      expect(r.encoding).toBe('shift_jis');
      expect(r.text).toContain('本文は日本語です');
      expect(r.text).not.toContain('�');
    }
  });

  it('reads a pure UTF-8 file via the fast path', () => {
    const r = decodeFileText(Buffer.from(JP, 'utf-8'));
    expect(r).toEqual({ binary: false, encoding: 'utf-8', text: JP });
  });

  it('does not silently swallow a truncated trailing UTF-8 byte as empty text', () => {
    // Regression (codex): streaming decode returned {binary:false, text:''} for a
    // lone 0xE3; a complete valid body followed by a truncated byte must keep the body.
    const lone = decodeFileText(Buffer.from([0xe3]));
    expect(lone.binary === false && lone.text === '').toBe(false);

    const truncated = decodeFileText(Buffer.concat([Buffer.from('あいう', 'utf-8'), Buffer.from([0xe3])]));
    expect(truncated.binary).toBe(false);
    if (!truncated.binary) expect(truncated.text).toContain('あいう');
  });

  it('rejects binary hidden behind a > 8KB ASCII prefix (NUL in the tail)', () => {
    // Regression (codex): head-only detection saw pure ASCII and NUL is valid
    // UTF-8, so a NUL-bearing tail slipped through as text. Whole-file NUL check.
    const asciiHead = Buffer.from('a'.repeat(9000), 'ascii');
    const nulTail = Buffer.from([0x00, 0x01, 0x02, 0x00, 0xff, 0x00]);
    expect(decodeFileText(Buffer.concat([asciiHead, nulTail])).binary).toBe(true);
  });

  it('preserves a UTF-8 BOM through decode/encode (labels it utf-8-bom)', () => {
    const withBom = Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from('見出し\n本文', 'utf-8')]);
    const r = decodeFileText(withBom);
    expect(r.binary).toBe(false);
    if (!r.binary) {
      expect(r.encoding).toBe('utf-8-bom');
      expect(r.text.charCodeAt(0)).not.toBe(0xfeff); // BOM stripped from the text view
      const reencoded = encodeFileText(r.text, r.encoding)!;
      expect([reencoded[0], reencoded[1], reencoded[2]]).toEqual([0xef, 0xbb, 0xbf]);
    }
  });

  it('rejects a magic-byte binary whose bytes are valid UTF-8 (PDF/ZIP header)', () => {
    // '%PDF-1.4\nneedle\n' is valid UTF-8, but the magic byte must win over the
    // UTF-8 fast path so Grep does not leak it.
    expect(decodeFileText(Buffer.from('%PDF-1.4\nneedle\n', 'ascii')).binary).toBe(true);
    expect(decodeFileText(Buffer.from([0x50, 0x4b, 0x03, 0x04, 0x6e, 0x65, 0x65, 0x64])).binary).toBe(true);
  });
});

describe('encodeFileText (round-trip guard)', () => {
  it('returns bytes when the text is representable in the target encoding', () => {
    const bytes = encodeFileText('日本語テキスト\n', 'shift_jis');
    expect(bytes).not.toBeNull();
    expect(iconv.decode(bytes!, 'Shift_JIS')).toBe('日本語テキスト\n');
  });

  it('returns null when the text has chars the encoding cannot represent (emoji into Shift_JIS)', () => {
    expect(encodeFileText('絵文字😀を含む\n', 'shift_jis')).toBeNull();
  });

  it('re-adds the BOM when encoding UTF-16 so the file stays readable next time', () => {
    const le = encodeFromUtf8('日本語X\n', 'utf-16le');
    expect([le[0], le[1]]).toEqual([0xff, 0xfe]);
    const be = encodeFromUtf8('日本語X\n', 'utf-16be');
    expect([be[0], be[1]]).toEqual([0xfe, 0xff]);
    // Round-trips through decodeFileText as UTF-16, not rejected as binary.
    const back = decodeFileText(le);
    expect(back.binary).toBe(false);
    if (!back.binary) expect(back.text.replace(/^/, '')).toBe('日本語X\n');
  });
});