sync: update from private repo (dfadcd5f)

2026-06-23 06:38:48 +00:00
parent 6a2f2cc736
commit 29ccaf1e92
377 changed files with 31028 additions and 8994 deletions
@@ -0,0 +1,121 @@
+import { describe, it, expect } from 'vitest';
+import { extractReadableMarkdown, truncateForAgent } from './readability.js';
+
+const ARTICLE_HTML = `<!DOCTYPE html>
+<html><head><title>Test Article Title</title></head>
+<body>
+  <nav><a href="/">Home</a><a href="/about">About</a><a href="/contact">Contact</a></nav>
+  <header>SITE BANNER — SUBSCRIBE NOW — ADVERTISEMENT</header>
+  <aside>Related links sidebar junk and tracking pixels</aside>
+  <article>
+    <h1>The Main Heading</h1>
+    <p>The first paragraph of the article body, with a <a href="https://example.com/page">helpful link</a> inside it. This needs to be long enough that readability accepts it as real content rather than boilerplate noise.</p>
+    <h2>A Subsection</h2>
+    <ul><li>first bullet point</li><li>second bullet point</li><li>third bullet point</li></ul>
+    <table>
+      <thead><tr><th>Name</th><th>Value</th></tr></thead>
+      <tbody><tr><td>alpha</td><td>1</td></tr><tr><td>beta</td><td>2</td></tr></tbody>
+    </table>
+    <p>A closing paragraph with sufficient length so the readability heuristics treat this document as a genuine article and extract the body cleanly. Lorem ipsum dolor sit amet, consectetur adipiscing elit.</p>
+  </article>
+  <footer>FOOTER COPYRIGHT — PRIVACY POLICY — TERMS</footer>
+</body></html>`;
+
+describe('extractReadableMarkdown', () => {
+  it('記事本文を抽出しボイラープレートを除去する', () => {
+    const result = extractReadableMarkdown(ARTICLE_HTML);
+    expect(result).not.toBeNull();
+    const md = result!.markdown;
+    // 本文は残る
+    expect(md).toContain('first paragraph of the article body');
+    expect(md).toContain('closing paragraph');
+    // ナビ・広告・フッターは除去される
+    expect(md).not.toContain('SITE BANNER');
+    expect(md).not.toContain('FOOTER COPYRIGHT');
+    expect(md).not.toContain('PRIVACY POLICY');
+  });
+
+  it('見出しが Markdown の atx 記法で残る', () => {
+    const md = extractReadableMarkdown(ARTICLE_HTML)!.markdown;
+    expect(md).toMatch(/#+\s+The Main Heading/);
+    expect(md).toMatch(/#+\s+A Subsection/);
+  });
+
+  it('リストが Markdown 箇条書きで残る', () => {
+    const md = extractReadableMarkdown(ARTICLE_HTML)!.markdown;
+    expect(md).toMatch(/-\s+first bullet point/);
+    expect(md).toMatch(/-\s+second bullet point/);
+  });
+
+  it('テーブルが GFM テーブル記法で残る', () => {
+    const md = extractReadableMarkdown(ARTICLE_HTML)!.markdown;
+    expect(md).toContain('| Name | Value |');
+    expect(md).toContain('| --- | --- |');
+    expect(md).toMatch(/\| alpha \| 1 \|/);
+  });
+
+  it('リンクが Markdown リンク記法で残る', () => {
+    const md = extractReadableMarkdown(ARTICLE_HTML)!.markdown;
+    expect(md).toContain('[helpful link](https://example.com/page)');
+  });
+
+  it('タイトルを抽出する', () => {
+    const result = extractReadableMarkdown(ARTICLE_HTML);
+    expect(result!.title).toBe('Test Article Title');
+  });
+
+  it('空 HTML では null を返す（フォールバック用）', () => {
+    expect(extractReadableMarkdown('')).toBeNull();
+    expect(extractReadableMarkdown('   ')).toBeNull();
+  });
+
+  it('本文のないガベージ HTML では null を返す', () => {
+    const garbage = '<html><body><div></div><span></span></body></html>';
+    expect(extractReadableMarkdown(garbage)).toBeNull();
+  });
+
+  it('selector 指定時はそのサブツリーだけを抽出する', () => {
+    const html = `<html><body>
+      <div id="wanted"><h2>Picked Section</h2><p>only this part should be extracted here</p></div>
+      <div id="other"><p>this other part must NOT appear in the output</p></div>
+    </body></html>`;
+    const result = extractReadableMarkdown(html, { selector: '#wanted' });
+    expect(result).not.toBeNull();
+    expect(result!.markdown).toContain('Picked Section');
+    expect(result!.markdown).toContain('only this part');
+    expect(result!.markdown).not.toContain('other part');
+  });
+
+  it('selector が見つからなければ null を返す（フォールバック用）', () => {
+    expect(extractReadableMarkdown(ARTICLE_HTML, { selector: '#does-not-exist' })).toBeNull();
+  });
+
+  it('baseUrl 指定時にルート相対リンクを絶対化する', () => {
+    const html = `<html><body><article>
+      <h1>Heading For Length</h1>
+      <p>A paragraph long enough to pass readability with a <a href="/docs/page">root relative link</a> embedded so we can verify absolutization works correctly here. Lorem ipsum dolor sit amet consectetur.</p>
+    </article></body></html>`;
+    const md = extractReadableMarkdown(html, { baseUrl: 'https://site.example/blog/post' })!.markdown;
+    expect(md).toContain('https://site.example/docs/page');
+    expect(md).not.toContain('](/docs/page)');
+  });
+});
+
+describe('truncateForAgent', () => {
+  it('上限以下ならそのまま返す', () => {
+    expect(truncateForAgent('short', 100)).toBe('short');
+  });
+
+  it('上限超過で切り詰め、source 保存済みなら full 版の在処を案内する', () => {
+    const long = 'x'.repeat(50);
+    const out = truncateForAgent(long, 10, 'article.md');
+    expect(out.startsWith('x'.repeat(10))).toBe(true);
+    expect(out).toContain('full version saved to source/article.md');
+  });
+
+  it('上限超過で source 未保存なら素の truncated note を付ける', () => {
+    const out = truncateForAgent('y'.repeat(50), 10, null);
+    expect(out).toContain('... (truncated)');
+    expect(out).not.toContain('saved to source');
+  });
+});