This commit is contained in:
oss-sync
@@ -0,0 +1,287 @@
|
||||
/**
|
||||
* Encoding-aware text layer on top of the pure byte sniffer in binary-detect.ts.
|
||||
*
|
||||
* binary-detect.ts answers a binary yes/no and recognizes UTF-8 / BOM-tagged
|
||||
* UTF-16 only — every other text encoding (Shift_JIS/CP932, EUC-JP, ISO-2022-JP,
|
||||
* windows-1252/Latin-1, …) trips its strict-UTF-8 gate and is reported as
|
||||
* `utf8-decode-fail`, i.e. "binary". That is wrong for the common case of a
|
||||
* Japanese .txt / CSV saved on Windows (CP932).
|
||||
*
|
||||
* This module recovers those: when the sniffer's only complaint is
|
||||
* `utf8-decode-fail`, it trial-decodes a short ladder of candidate encodings
|
||||
* with iconv-lite and picks the one that produces the least mojibake. Callers
|
||||
* (Read/Grep/Edit) then transcode the whole file to UTF-8 for the LLM while
|
||||
* keeping the raw bytes so edits round-trip back to the original encoding.
|
||||
*
|
||||
* iconv-lite is pure JS (no ICU dependency), so Shift_JIS/EUC-JP decoding works
|
||||
* the same on small-icu Node builds where the native TextDecoder lacks those
|
||||
* labels. Only UTF-8 and BOM-tagged UTF-16 stay on the built-in decoder.
|
||||
*/
|
||||
|
||||
import iconv from 'iconv-lite';
|
||||
import {
|
||||
looksLikeBinaryBytes,
|
||||
controlCharRatio,
|
||||
CONTROL_CHAR_RATIO_THRESHOLD,
|
||||
SNIFF_HEAD_BYTES,
|
||||
stripLeadingBom,
|
||||
} from './binary-detect.js';
|
||||
|
||||
const UTF16LE_BOM = Buffer.from([0xff, 0xfe]);
|
||||
const UTF16BE_BOM = Buffer.from([0xfe, 0xff]);
|
||||
const UTF8_BOM = Buffer.from([0xef, 0xbb, 0xbf]);
|
||||
|
||||
// Internal encoding label for UTF-8 files that carry a leading BOM. TextDecoder
|
||||
// strips the BOM on read, so without tracking it separately an Edit/Write would
|
||||
// silently drop it and break Windows tools that expect it. Treated as UTF-8
|
||||
// everywhere except that the BOM is re-added on encode.
|
||||
export const UTF8_BOM_LABEL = 'utf-8-bom';
|
||||
|
||||
function hasUtf8Bom(buf: Buffer): boolean {
|
||||
return buf.length >= 3 && buf[0] === 0xef && buf[1] === 0xbb && buf[2] === 0xbf;
|
||||
}
|
||||
|
||||
/**
|
||||
* Strict UTF-8 decode of a COMPLETE buffer. Unlike binary-detect's streaming
|
||||
* `decodeText` (used for 8 KB head sniffing, where a multibyte char cut at the
|
||||
* sample boundary must be tolerated), this flushes at EOF, so a genuinely
|
||||
* truncated trailing sequence is fatal instead of being silently dropped.
|
||||
*/
|
||||
function strictUtf8Whole(buf: Buffer): string | null {
|
||||
try {
|
||||
return new TextDecoder('utf-8', { fatal: true }).decode(buf);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export type TextEncodingVerdict =
|
||||
| { binary: true; reason: string }
|
||||
| { binary: false; encoding: string };
|
||||
|
||||
export type DecodedFile =
|
||||
| { binary: true; reason: string }
|
||||
| { binary: false; encoding: string; text: string };
|
||||
|
||||
/**
|
||||
* Candidate encodings tried, in priority order, when a buffer is not valid
|
||||
* UTF-8 but also shows no hard binary signal. Japanese-first because that is
|
||||
* the dominant real case (Windows CP932). windows-1252 is last: it decodes
|
||||
* almost any byte, so it must only win when the multibyte candidates all fail
|
||||
* (strict `<` tie-break below keeps the earliest candidate on equal scores).
|
||||
*/
|
||||
// Japanese-first, and deliberately Japanese-only among the multibyte codecs:
|
||||
// GBK/Big5 (Chinese) would grab ambiguous kana/halfwidth Japanese and mojibake
|
||||
// it. windows-1252 last as the single-byte Western fallback (see the tie-break).
|
||||
const CANDIDATE_ENCODINGS = ['Shift_JIS', 'EUC-JP', 'ISO-2022-JP', 'windows-1252'];
|
||||
|
||||
/** Accept a trial decode only when at most this fraction of chars are garbage. */
|
||||
const MOJIBAKE_ACCEPT_RATIO = 0.02;
|
||||
|
||||
/**
|
||||
* A near-zero mojibake score is not enough on its own: windows-1252 maps almost
|
||||
* every byte and multibyte CJK encodings (GBK/Big5) accept many random byte
|
||||
* pairs, so high-entropy binary can decode "cleanly" into a soup of symbols and
|
||||
* rare ideographs. Require that the decoded chars are also *predominantly*
|
||||
* ordinary text — ASCII, Japanese kana/kanji, CJK punctuation, fullwidth forms,
|
||||
* or Latin accented letters. Real text scores ~1.0; binary-as-cp1252/GBK scores
|
||||
* well below this because it is dominated by symbol/rare-block code points.
|
||||
*/
|
||||
const PLAUSIBLE_TEXT_MIN = 0.8;
|
||||
|
||||
const REPLACEMENT_CHAR = 0xfffd;
|
||||
|
||||
/** Fraction of decoded chars that are U+FFFD or non-whitespace C0/C1 controls. */
|
||||
function mojibakeRatio(decoded: string): number {
|
||||
if (decoded.length === 0) return 1;
|
||||
let bad = 0;
|
||||
for (let i = 0; i < decoded.length; i++) {
|
||||
const code = decoded.charCodeAt(i);
|
||||
if (code === REPLACEMENT_CHAR) {
|
||||
bad++;
|
||||
} else if (
|
||||
(code <= 0x08 || code === 0x0b || code === 0x0c || (code >= 0x0e && code <= 0x1f) ||
|
||||
code === 0x7f || (code >= 0x80 && code <= 0x9f))
|
||||
) {
|
||||
bad++;
|
||||
}
|
||||
}
|
||||
return bad / decoded.length;
|
||||
}
|
||||
|
||||
/** True for code points that appear in ordinary human text (see PLAUSIBLE_TEXT_MIN). */
|
||||
function isPlausibleTextChar(code: number): boolean {
|
||||
return (
|
||||
code === 0x09 || code === 0x0a || code === 0x0d || // tab / newlines
|
||||
(code >= 0x20 && code <= 0x7e) || // ASCII printable
|
||||
(code >= 0x00c0 && code <= 0x00ff) || // Latin-1 accented letters
|
||||
(code >= 0x3000 && code <= 0x303f) || // CJK symbols & punctuation
|
||||
(code >= 0x3040 && code <= 0x309f) || // Hiragana
|
||||
(code >= 0x30a0 && code <= 0x30ff) || // Katakana
|
||||
(code >= 0x4e00 && code <= 0x9fff) || // CJK Unified Ideographs
|
||||
(code >= 0xff00 && code <= 0xffef) // Fullwidth / halfwidth forms
|
||||
);
|
||||
}
|
||||
|
||||
/** Fraction of decoded chars that are ordinary text (not stray symbols/rare blocks). */
|
||||
function plausibleTextRatio(decoded: string): number {
|
||||
if (decoded.length === 0) return 0;
|
||||
let ok = 0;
|
||||
for (let i = 0; i < decoded.length; i++) {
|
||||
if (isPlausibleTextChar(decoded.charCodeAt(i))) ok++;
|
||||
}
|
||||
return ok / decoded.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fraction of chars that are *body-text* Japanese/Latin — like isPlausibleTextChar
|
||||
* but EXCLUDING halfwidth katakana (U+FF61–FF9F) and Latin-1 symbols. Used only to
|
||||
* break ties between candidates: EUC-JP kana decoded (wrongly) as Shift_JIS lands
|
||||
* in halfwidth katakana, so the candidate whose decode is richer in fullwidth
|
||||
* kana/kanji is the right one. Two encodings can both hit zero mojibake, so this
|
||||
* secondary key is what stops EUC-JP text from being mislabeled Shift_JIS.
|
||||
*/
|
||||
function preferredTextRatio(decoded: string): number {
|
||||
if (decoded.length === 0) return 0;
|
||||
let ok = 0;
|
||||
for (let i = 0; i < decoded.length; i++) {
|
||||
const code = decoded.charCodeAt(i);
|
||||
if (
|
||||
code === 0x09 || code === 0x0a || code === 0x0d ||
|
||||
(code >= 0x20 && code <= 0x7e) || // ASCII printable
|
||||
(code >= 0x00c0 && code <= 0x00ff) || // Latin-1 accented letters
|
||||
(code >= 0x3000 && code <= 0x303f) || // CJK symbols & punctuation
|
||||
(code >= 0x3040 && code <= 0x309f) || // Hiragana
|
||||
(code >= 0x30a0 && code <= 0x30ff) || // full Katakana
|
||||
(code >= 0x4e00 && code <= 0x9fff) || // CJK Unified Ideographs
|
||||
(code >= 0xff01 && code <= 0xff60) // Fullwidth forms (not halfwidth)
|
||||
) {
|
||||
ok++;
|
||||
}
|
||||
}
|
||||
return ok / decoded.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extend the pure binary sniffer with non-UTF-8 text recovery.
|
||||
* Returns a concrete encoding label callers pass to decodeToUtf8.
|
||||
*/
|
||||
export function detectTextEncoding(head: Buffer): TextEncodingVerdict {
|
||||
const base = looksLikeBinaryBytes(head);
|
||||
if (!base.binary) return { binary: false, encoding: base.encoding };
|
||||
|
||||
// Only a strict-UTF-8 failure is potentially recoverable text. A magic-byte
|
||||
// signature, an embedded NUL, or a high control-char ratio is a hard binary
|
||||
// signal — leave those rejected exactly as before.
|
||||
if (base.reason !== 'utf8-decode-fail') return base;
|
||||
|
||||
// utf8-decode-fail returns before binary-detect's own control-ratio gate, so
|
||||
// re-check it here: high control density means binary, not exotic-encoded text.
|
||||
if (controlCharRatio(head) > CONTROL_CHAR_RATIO_THRESHOLD) {
|
||||
return { binary: true, reason: base.reason };
|
||||
}
|
||||
|
||||
let best: string | null = null;
|
||||
let bestScore = Infinity;
|
||||
let bestMojibake = 1;
|
||||
let bestPlausible = 0;
|
||||
for (const enc of CANDIDATE_ENCODINGS) {
|
||||
if (!iconv.encodingExists(enc)) continue;
|
||||
const decoded = iconv.decode(head, enc);
|
||||
const mojibake = mojibakeRatio(decoded);
|
||||
// Primary key: mojibake (×10 so it dominates). Secondary: prefer the decode
|
||||
// richer in body-text Japanese/Latin, which separates true EUC-JP from the
|
||||
// same bytes decoded as Shift_JIS halfwidth katakana.
|
||||
const score = mojibake * 10 + (1 - preferredTextRatio(decoded));
|
||||
if (score < bestScore) {
|
||||
bestScore = score;
|
||||
best = enc;
|
||||
bestMojibake = mojibake;
|
||||
bestPlausible = plausibleTextRatio(decoded);
|
||||
}
|
||||
}
|
||||
|
||||
if (best && bestMojibake <= MOJIBAKE_ACCEPT_RATIO && bestPlausible >= PLAUSIBLE_TEXT_MIN) {
|
||||
return { binary: false, encoding: best.toLowerCase() };
|
||||
}
|
||||
return { binary: true, reason: base.reason };
|
||||
}
|
||||
|
||||
/** Decode raw file bytes of a known encoding into a UTF-8 JS string. */
|
||||
export function decodeToUtf8(buf: Buffer, encoding: string): string {
|
||||
if (encoding === 'utf-8') return buf.toString('utf-8');
|
||||
if (encoding === UTF8_BOM_LABEL) return stripLeadingBom(buf.toString('utf-8'));
|
||||
return iconv.decode(buf, encoding);
|
||||
}
|
||||
|
||||
/**
|
||||
* Encode an (edited) UTF-8 string back into bytes of the original encoding.
|
||||
*
|
||||
* For UTF-16 the BOM is re-added: the encoding label is only produced for
|
||||
* BOM-tagged files (iconv strips the BOM on decode and does not re-add it on
|
||||
* encode), and a BOMless UTF-16 file would be rejected as binary on the next
|
||||
* Read (leading NUL). Re-adding it keeps the round-trip readable.
|
||||
*/
|
||||
export function encodeFromUtf8(text: string, encoding: string): Buffer {
|
||||
if (encoding === 'utf-8') return Buffer.from(text, 'utf-8');
|
||||
if (encoding === UTF8_BOM_LABEL) return Buffer.concat([UTF8_BOM, Buffer.from(text, 'utf-8')]);
|
||||
const body = iconv.encode(text, encoding);
|
||||
if (encoding === 'utf-16le') return Buffer.concat([UTF16LE_BOM, body]);
|
||||
if (encoding === 'utf-16be') return Buffer.concat([UTF16BE_BOM, body]);
|
||||
return body;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode a WHOLE file buffer to a UTF-8 string, choosing the encoding.
|
||||
*
|
||||
* Detection runs over the entire buffer, not just the head: a file whose first
|
||||
* 8 KiB is pure ASCII (a long English header/comment) but whose tail is
|
||||
* Shift_JIS must not be mistaken for UTF-8 and mangled to U+FFFD. The fast path
|
||||
* is a single whole-file strict UTF-8 decode; only when that fails do we run the
|
||||
* candidate ladder.
|
||||
*/
|
||||
export function decodeFileText(full: Buffer): DecodedFile {
|
||||
// Hard binary signals (magic byte / NUL / high control ratio) MUST be checked
|
||||
// before any UTF-8 fast path: a PDF/ZIP/GIF header can be valid UTF-8, and NUL
|
||||
// is itself valid UTF-8, so trusting the fast path would let Grep leak binary.
|
||||
const head = full.length > SNIFF_HEAD_BYTES ? full.subarray(0, SNIFF_HEAD_BYTES) : full;
|
||||
const headVerdict = detectTextEncoding(head);
|
||||
if (headVerdict.binary) return headVerdict;
|
||||
|
||||
// BOM-tagged UTF-16 legitimately contains NUL bytes; decode via its encoding
|
||||
// and skip the NUL gate below (which would otherwise reject it as binary).
|
||||
if (headVerdict.encoding === 'utf-16le' || headVerdict.encoding === 'utf-16be') {
|
||||
return { binary: false, encoding: headVerdict.encoding, text: decodeToUtf8(full, headVerdict.encoding) };
|
||||
}
|
||||
|
||||
// Whole-file hard-binary check: an ASCII head can hide NUL / high control ratio
|
||||
// in the tail (the head-only verdict above cannot see it).
|
||||
if (full.includes(0)) return { binary: true, reason: 'nul-byte' };
|
||||
if (controlCharRatio(full) > CONTROL_CHAR_RATIO_THRESHOLD) {
|
||||
return { binary: true, reason: `control-ratio:${controlCharRatio(full).toFixed(2)}` };
|
||||
}
|
||||
|
||||
// Whole-file UTF-8 validation (an ASCII prefix must not hide a Shift_JIS tail;
|
||||
// a truncated trailing multibyte must not be silently dropped either). Track a
|
||||
// leading BOM so Edit/Write can re-add it.
|
||||
const utf8 = strictUtf8Whole(full);
|
||||
if (utf8 !== null) {
|
||||
return { binary: false, encoding: hasUtf8Bom(full) ? UTF8_BOM_LABEL : 'utf-8', text: utf8 };
|
||||
}
|
||||
|
||||
const verdict = detectTextEncoding(full);
|
||||
if (verdict.binary) return verdict;
|
||||
return { binary: false, encoding: verdict.encoding, text: decodeToUtf8(full, verdict.encoding) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Encode a UTF-8 string back to bytes of `encoding`, but only if it round-trips.
|
||||
* iconv-lite silently substitutes unrepresentable chars (e.g. an emoji written
|
||||
* into a Shift_JIS file) with '?', which would corrupt the save. Returns null so
|
||||
* the caller can refuse rather than write bytes that differ from the request.
|
||||
*/
|
||||
export function encodeFileText(text: string, encoding: string): Buffer | null {
|
||||
const bytes = encodeFromUtf8(text, encoding);
|
||||
if (encoding === 'utf-8') return bytes;
|
||||
return decodeToUtf8(bytes, encoding) === text ? bytes : null;
|
||||
}
|
||||
Reference in New Issue
Block a user