/**
 * Encoding-aware text layer on top of the pure byte sniffer in binary-detect.ts.
 *
 * binary-detect.ts answers a binary yes/no and recognizes UTF-8 / BOM-tagged
 * UTF-16 only — every other text encoding (Shift_JIS/CP932, EUC-JP, ISO-2022-JP,
 * windows-1252/Latin-1, …) trips its strict-UTF-8 gate and is reported as
 * `utf8-decode-fail`, i.e. "binary". That is wrong for the common case of a
 * Japanese .txt / CSV saved on Windows (CP932).
 *
 * This module recovers those: when the sniffer's only complaint is
 * `utf8-decode-fail`, it trial-decodes a short ladder of candidate encodings
 * with iconv-lite and picks the one that produces the least mojibake. Callers
 * (Read/Grep/Edit) then transcode the whole file to UTF-8 for the LLM while
 * keeping the raw bytes so edits round-trip back to the original encoding.
 *
 * iconv-lite is pure JS (no ICU dependency), so Shift_JIS/EUC-JP decoding works
 * the same on small-icu Node builds where the native TextDecoder lacks those
 * labels. Only UTF-8 and BOM-tagged UTF-16 stay on the built-in decoder.
 */

import iconv from 'iconv-lite';
import {
  looksLikeBinaryBytes,
  controlCharRatio,
  CONTROL_CHAR_RATIO_THRESHOLD,
  SNIFF_HEAD_BYTES,
  stripLeadingBom,
} from './binary-detect.js';

const UTF16LE_BOM = Buffer.from([0xff, 0xfe]);
const UTF16BE_BOM = Buffer.from([0xfe, 0xff]);
const UTF8_BOM = Buffer.from([0xef, 0xbb, 0xbf]);

// Internal encoding label for UTF-8 files that carry a leading BOM. TextDecoder
// strips the BOM on read, so without tracking it separately an Edit/Write would
// silently drop it and break Windows tools that expect it. Treated as UTF-8
// everywhere except that the BOM is re-added on encode.
export const UTF8_BOM_LABEL = 'utf-8-bom';

function hasUtf8Bom(buf: Buffer): boolean {
  return buf.length >= 3 && buf[0] === 0xef && buf[1] === 0xbb && buf[2] === 0xbf;
}

/**
 * Strict UTF-8 decode of a COMPLETE buffer. Unlike binary-detect's streaming
 * `decodeText` (used for 8 KB head sniffing, where a multibyte char cut at the
 * sample boundary must be tolerated), this flushes at EOF, so a genuinely
 * truncated trailing sequence is fatal instead of being silently dropped.
 */
function strictUtf8Whole(buf: Buffer): string | null {
  try {
    return new TextDecoder('utf-8', { fatal: true }).decode(buf);
  } catch {
    return null;
  }
}

export type TextEncodingVerdict =
  | { binary: true; reason: string }
  | { binary: false; encoding: string };

export type DecodedFile =
  | { binary: true; reason: string }
  | { binary: false; encoding: string; text: string };

/**
 * Candidate encodings tried, in priority order, when a buffer is not valid
 * UTF-8 but also shows no hard binary signal. Japanese-first because that is
 * the dominant real case (Windows CP932). windows-1252 is last: it decodes
 * almost any byte, so it must only win when the multibyte candidates all fail
 * (strict `<` tie-break below keeps the earliest candidate on equal scores).
 */
// Japanese-first, and deliberately Japanese-only among the multibyte codecs:
// GBK/Big5 (Chinese) would grab ambiguous kana/halfwidth Japanese and mojibake
// it. windows-1252 last as the single-byte Western fallback (see the tie-break).
const CANDIDATE_ENCODINGS = ['Shift_JIS', 'EUC-JP', 'ISO-2022-JP', 'windows-1252'];

/** Accept a trial decode only when at most this fraction of chars are garbage. */
const MOJIBAKE_ACCEPT_RATIO = 0.02;

/**
 * A near-zero mojibake score is not enough on its own: windows-1252 maps almost
 * every byte and multibyte CJK encodings (GBK/Big5) accept many random byte
 * pairs, so high-entropy binary can decode "cleanly" into a soup of symbols and
 * rare ideographs. Require that the decoded chars are also *predominantly*
 * ordinary text — ASCII, Japanese kana/kanji, CJK punctuation, fullwidth forms,
 * or Latin accented letters. Real text scores ~1.0; binary-as-cp1252/GBK scores
 * well below this because it is dominated by symbol/rare-block code points.
 */
const PLAUSIBLE_TEXT_MIN = 0.8;

const REPLACEMENT_CHAR = 0xfffd;

/** Fraction of decoded chars that are U+FFFD or non-whitespace C0/C1 controls. */
function mojibakeRatio(decoded: string): number {
  if (decoded.length === 0) return 1;
  let bad = 0;
  for (let i = 0; i < decoded.length; i++) {
    const code = decoded.charCodeAt(i);
    if (code === REPLACEMENT_CHAR) {
      bad++;
    } else if (
      (code <= 0x08 || code === 0x0b || code === 0x0c || (code >= 0x0e && code <= 0x1f) ||
        code === 0x7f || (code >= 0x80 && code <= 0x9f))
    ) {
      bad++;
    }
  }
  return bad / decoded.length;
}

/** True for code points that appear in ordinary human text (see PLAUSIBLE_TEXT_MIN). */
function isPlausibleTextChar(code: number): boolean {
  return (
    code === 0x09 || code === 0x0a || code === 0x0d || // tab / newlines
    (code >= 0x20 && code <= 0x7e) ||                  // ASCII printable
    (code >= 0x00c0 && code <= 0x00ff) ||              // Latin-1 accented letters
    (code >= 0x3000 && code <= 0x303f) ||              // CJK symbols & punctuation
    (code >= 0x3040 && code <= 0x309f) ||              // Hiragana
    (code >= 0x30a0 && code <= 0x30ff) ||              // Katakana
    (code >= 0x4e00 && code <= 0x9fff) ||              // CJK Unified Ideographs
    (code >= 0xff00 && code <= 0xffef)                 // Fullwidth / halfwidth forms
  );
}

/** Fraction of decoded chars that are ordinary text (not stray symbols/rare blocks). */
function plausibleTextRatio(decoded: string): number {
  if (decoded.length === 0) return 0;
  let ok = 0;
  for (let i = 0; i < decoded.length; i++) {
    if (isPlausibleTextChar(decoded.charCodeAt(i))) ok++;
  }
  return ok / decoded.length;
}

/**
 * Fraction of chars that are *body-text* Japanese/Latin — like isPlausibleTextChar
 * but EXCLUDING halfwidth katakana (U+FF61–FF9F) and Latin-1 symbols. Used only to
 * break ties between candidates: EUC-JP kana decoded (wrongly) as Shift_JIS lands
 * in halfwidth katakana, so the candidate whose decode is richer in fullwidth
 * kana/kanji is the right one. Two encodings can both hit zero mojibake, so this
 * secondary key is what stops EUC-JP text from being mislabeled Shift_JIS.
 */
function preferredTextRatio(decoded: string): number {
  if (decoded.length === 0) return 0;
  let ok = 0;
  for (let i = 0; i < decoded.length; i++) {
    const code = decoded.charCodeAt(i);
    if (
      code === 0x09 || code === 0x0a || code === 0x0d ||
      (code >= 0x20 && code <= 0x7e) ||                  // ASCII printable
      (code >= 0x00c0 && code <= 0x00ff) ||              // Latin-1 accented letters
      (code >= 0x3000 && code <= 0x303f) ||              // CJK symbols & punctuation
      (code >= 0x3040 && code <= 0x309f) ||              // Hiragana
      (code >= 0x30a0 && code <= 0x30ff) ||              // full Katakana
      (code >= 0x4e00 && code <= 0x9fff) ||              // CJK Unified Ideographs
      (code >= 0xff01 && code <= 0xff60)                 // Fullwidth forms (not halfwidth)
    ) {
      ok++;
    }
  }
  return ok / decoded.length;
}

/**
 * Extend the pure binary sniffer with non-UTF-8 text recovery.
 * Returns a concrete encoding label callers pass to decodeToUtf8.
 */
export function detectTextEncoding(head: Buffer): TextEncodingVerdict {
  const base = looksLikeBinaryBytes(head);
  if (!base.binary) return { binary: false, encoding: base.encoding };

  // Only a strict-UTF-8 failure is potentially recoverable text. A magic-byte
  // signature, an embedded NUL, or a high control-char ratio is a hard binary
  // signal — leave those rejected exactly as before.
  if (base.reason !== 'utf8-decode-fail') return base;

  // utf8-decode-fail returns before binary-detect's own control-ratio gate, so
  // re-check it here: high control density means binary, not exotic-encoded text.
  if (controlCharRatio(head) > CONTROL_CHAR_RATIO_THRESHOLD) {
    return { binary: true, reason: base.reason };
  }

  let best: string | null = null;
  let bestScore = Infinity;
  let bestMojibake = 1;
  let bestPlausible = 0;
  for (const enc of CANDIDATE_ENCODINGS) {
    if (!iconv.encodingExists(enc)) continue;
    const decoded = iconv.decode(head, enc);
    const mojibake = mojibakeRatio(decoded);
    // Primary key: mojibake (×10 so it dominates). Secondary: prefer the decode
    // richer in body-text Japanese/Latin, which separates true EUC-JP from the
    // same bytes decoded as Shift_JIS halfwidth katakana.
    const score = mojibake * 10 + (1 - preferredTextRatio(decoded));
    if (score < bestScore) {
      bestScore = score;
      best = enc;
      bestMojibake = mojibake;
      bestPlausible = plausibleTextRatio(decoded);
    }
  }

  if (best && bestMojibake <= MOJIBAKE_ACCEPT_RATIO && bestPlausible >= PLAUSIBLE_TEXT_MIN) {
    return { binary: false, encoding: best.toLowerCase() };
  }
  return { binary: true, reason: base.reason };
}

/** Decode raw file bytes of a known encoding into a UTF-8 JS string. */
export function decodeToUtf8(buf: Buffer, encoding: string): string {
  if (encoding === 'utf-8') return buf.toString('utf-8');
  if (encoding === UTF8_BOM_LABEL) return stripLeadingBom(buf.toString('utf-8'));
  return iconv.decode(buf, encoding);
}

/**
 * Encode an (edited) UTF-8 string back into bytes of the original encoding.
 *
 * For UTF-16 the BOM is re-added: the encoding label is only produced for
 * BOM-tagged files (iconv strips the BOM on decode and does not re-add it on
 * encode), and a BOMless UTF-16 file would be rejected as binary on the next
 * Read (leading NUL). Re-adding it keeps the round-trip readable.
 */
export function encodeFromUtf8(text: string, encoding: string): Buffer {
  if (encoding === 'utf-8') return Buffer.from(text, 'utf-8');
  if (encoding === UTF8_BOM_LABEL) return Buffer.concat([UTF8_BOM, Buffer.from(text, 'utf-8')]);
  const body = iconv.encode(text, encoding);
  if (encoding === 'utf-16le') return Buffer.concat([UTF16LE_BOM, body]);
  if (encoding === 'utf-16be') return Buffer.concat([UTF16BE_BOM, body]);
  return body;
}

/**
 * Decode a WHOLE file buffer to a UTF-8 string, choosing the encoding.
 *
 * Detection runs over the entire buffer, not just the head: a file whose first
 * 8 KiB is pure ASCII (a long English header/comment) but whose tail is
 * Shift_JIS must not be mistaken for UTF-8 and mangled to U+FFFD. The fast path
 * is a single whole-file strict UTF-8 decode; only when that fails do we run the
 * candidate ladder.
 */
export function decodeFileText(full: Buffer): DecodedFile {
  // Hard binary signals (magic byte / NUL / high control ratio) MUST be checked
  // before any UTF-8 fast path: a PDF/ZIP/GIF header can be valid UTF-8, and NUL
  // is itself valid UTF-8, so trusting the fast path would let Grep leak binary.
  const head = full.length > SNIFF_HEAD_BYTES ? full.subarray(0, SNIFF_HEAD_BYTES) : full;
  const headVerdict = detectTextEncoding(head);
  if (headVerdict.binary) return headVerdict;

  // BOM-tagged UTF-16 legitimately contains NUL bytes; decode via its encoding
  // and skip the NUL gate below (which would otherwise reject it as binary).
  if (headVerdict.encoding === 'utf-16le' || headVerdict.encoding === 'utf-16be') {
    return { binary: false, encoding: headVerdict.encoding, text: decodeToUtf8(full, headVerdict.encoding) };
  }

  // Whole-file hard-binary check: an ASCII head can hide NUL / high control ratio
  // in the tail (the head-only verdict above cannot see it).
  if (full.includes(0)) return { binary: true, reason: 'nul-byte' };
  if (controlCharRatio(full) > CONTROL_CHAR_RATIO_THRESHOLD) {
    return { binary: true, reason: `control-ratio:${controlCharRatio(full).toFixed(2)}` };
  }

  // Whole-file UTF-8 validation (an ASCII prefix must not hide a Shift_JIS tail;
  // a truncated trailing multibyte must not be silently dropped either). Track a
  // leading BOM so Edit/Write can re-add it.
  const utf8 = strictUtf8Whole(full);
  if (utf8 !== null) {
    return { binary: false, encoding: hasUtf8Bom(full) ? UTF8_BOM_LABEL : 'utf-8', text: utf8 };
  }

  const verdict = detectTextEncoding(full);
  if (verdict.binary) return verdict;
  return { binary: false, encoding: verdict.encoding, text: decodeToUtf8(full, verdict.encoding) };
}

/**
 * Encode a UTF-8 string back to bytes of `encoding`, but only if it round-trips.
 * iconv-lite silently substitutes unrepresentable chars (e.g. an emoji written
 * into a Shift_JIS file) with '?', which would corrupt the save. Returns null so
 * the caller can refuse rather than write bytes that differ from the request.
 */
export function encodeFileText(text: string, encoding: string): Buffer | null {
  const bytes = encodeFromUtf8(text, encoding);
  if (encoding === 'utf-8') return bytes;
  return decodeToUtf8(bytes, encoding) === text ? bytes : null;
}