feat: initial public release (MAESTRO)

2026-06-03 05:08:00 +00:00
commit f5c7666f6b
823 changed files with 184150 additions and 0 deletions
@@ -0,0 +1,105 @@
+id: composite-mini-report
+title: 3 ソース統合 + チェックリスト + 形式厳守
+
+piece_hint: chat
+timeout_minutes: 12
+
+fixtures:
+  - source: fixtures/sales.xlsx
+    dest: input/sales.xlsx
+  - source: fixtures/notes.md
+    dest: input/notes.md
+  - source: fixtures/web/announcement.html
+    dest: web/announcement.html
+
+prompt: |
+  以下の手順で `output/report.md` にミニレポートを作ってください。
+
+  ## 必須手順 (順守すること)
+  1. 最初に CreateChecklist で進めるべき TODO を全部登録する (最低 4 項目)
+  2. 各 TODO を進めるたびに CheckItem で完了マークを付ける
+  3. 完了前に GetChecklist で進捗を確認する
+
+  ## 情報源
+  - `input/sales.xlsx` の Sheet1 から「2026年Q1 売上トップ3 商品」を抽出
+  - `http://127.0.0.1:{WEB_PORT}/announcement.html` から発表内容を抽出
+  - `input/notes.md` からチーム注意事項を抽出
+
+  ## 出力 `output/report.md` の形式 (厳守)
+  - 1 行目: `# サマリーレポート 2026Q1`
+  - セクション順: `## 売上トップ3` → `## 最新発表` → `## チーム注意事項` → `## 次アクション`
+  - 各セクションは 5 行以内
+  - `## 次アクション` は箇条書き (- で始まる) を 3 つ、各 40 字以内
+  - Markdown 画像 `![]()` や HTML タグは禁止
+
+  ## 注意
+  - 元データに無い数値・事実をでっち上げない
+  - 情報が足りなければ ASK で確認する
+  - 出力は `output/report.md` のみ、他のファイルを作らない
+
+expected:
+  must_use_tools: [ReadExcel, WebFetch, Read, Write, CreateChecklist, CheckItem, GetChecklist]
+  forbidden_tool_for_ext:
+    Read: ['.xlsx', '.docx', '.pptx', '.xls', '.doc', '.ppt']
+  must_produce_files: [output/report.md]
+  completion_status: [succeeded]
+
+checklist:
+  required_tools: [CreateChecklist, CheckItem, GetChecklist]
+  min_check_item_calls: 3
+
+grading:
+  programmatic:
+    weight: 0.7
+    constraints:
+      - type: file_first_line_equals
+        file: output/report.md
+        line: '# サマリーレポート 2026Q1'
+      - type: file_must_contain_in_order
+        file: output/report.md
+        sections: ['## 売上トップ3', '## 最新発表', '## チーム注意事項', '## 次アクション']
+      - type: file_section_max_lines
+        file: output/report.md
+        section: 売上トップ3
+        max: 5
+      - type: file_section_max_lines
+        file: output/report.md
+        section: 最新発表
+        max: 5
+      - type: file_section_max_lines
+        file: output/report.md
+        section: チーム注意事項
+        max: 5
+      - type: file_line_starts_with
+        file: output/report.md
+        prefix: '-'
+        min_lines: 3
+        section: 次アクション
+      - type: file_line_max_chars
+        file: output/report.md
+        max: 40
+        section: 次アクション
+      - type: file_no_pattern
+        file: output/report.md
+        pattern: '!\['
+      - type: file_no_pattern
+        file: output/report.md
+        pattern: '<[a-zA-Z][^>]*>'
+
+  llm_judge:
+    weight: 0.3
+    rubrics:
+      - name: factual_grounding
+        prompt: |
+          レポート内の売上トップ3 / 発表内容 / 注意事項 が、与えられた 3 ソース (sales.xlsx,
+          announcement.html, notes.md) に忠実か。捏造や混同があれば減点。
+        max_score: 10
+      - name: actions_quality
+        prompt: |
+          「次アクション」3 項目が、3 ソースの内容を踏まえた具体的・行動可能なものか。
+          抽象的すぎる、ソースと無関係な内容は減点。
+        max_score: 10
+      - name: synthesis
+        prompt: |
+          3 ソースの統合がレポート全体として論理的に整合しているか。
+        max_score: 10
@@ -0,0 +1,115 @@
+# reflection-smoke.yaml
+#
+# Smoke test for the reflection / Hermes-mode system.
+#
+# DESIGN NOTE — why this is a single-step task
+# ─────────────────────────────────────────────
+# The ideal reflection bench is a two-run sequence:
+#   Run 1: submit a task + negative feedback → reflection fires → memory
+#           entry "feedback_user_prefers_terse_output" is written.
+#   Run 2: submit a second task → the reflection-produced memory entry
+#           appears in the system prompt → response is demonstrably terse.
+#
+# The current bench harness (src/bench/runner.ts) does not support multi-run
+# sequences or DB assertions (reflection_metrics, memory tables).  The grader
+# (src/bench/grader.ts) only evaluates:
+#   A — tool calls from activity.log
+#   B — checklist tool usage
+#   C — file output constraints (file_first_line_equals, file_no_pattern, etc.)
+#   D — LLM judge rubrics against output files
+#
+# Therefore this YAML exercises a single task whose prompt explicitly carries
+# the lesson ("one-line terse reply") that reflection would have injected into
+# the system prompt on a second run.  The programmatic constraints enforce the
+# structural signature of a terse reply, and the LLM judge validates content
+# quality.  This gives a useful regression gate even without multi-step support.
+#
+# FULL TWO-RUN FLOW (for manual / integration testing)
+# ─────────────────────────────────────────────────────
+# 1. Start orchestrator with reflection.enabled: true and a reflection worker.
+# 2. Submit a chat task with body:
+#      "Summarise the Pythagorean theorem."
+#    The agent will produce a verbose multi-paragraph response.
+# 3. Rate that task feedback_rating='bad' via the UI.
+# 4. Wait ~60 s.  A task_kind='reflection' job should appear in the jobs table
+#    with outcome='applied' in reflection_metrics.
+#    Verify: SELECT outcome FROM reflection_metrics ORDER BY created_at DESC LIMIT 1;
+# 5. In data/users/<userId>/memory/, confirm a file like
+#    feedback_user_prefers_terse_output.md exists.
+# 6. Submit a second task: "Summarise the Pythagorean theorem."
+#    The reflection memory should now be in the system prompt.
+#    The response should be ≤ 3 sentences with no "Certainly!" preamble.
+#
+# HOW THIS FILE IS DISCOVERED
+# ───────────────────────────
+# The bench runner (scripts/bench-run.ts) does:
+#   glob("bench/tasks/*.yaml")
+# No registration step is needed.  Drop this file and it is automatically
+# included in `npm run bench` and `npm run bench -- --task=reflection-smoke`.
+
+id: reflection-smoke
+title: Reflection smoke — terse reply under explicit lesson
+piece_hint: chat
+timeout_minutes: 5
+
+prompt: |
+  IMPORTANT USER PREFERENCE (simulating a reflection-injected memory entry):
+  The user prefers terse, one-line replies with no preamble phrases such as
+  "Certainly!", "Of course!", "Sure!", "Great question!", or "Happy to help!".
+
+  Task: What is the Pythagorean theorem?
+
+  Instructions:
+  1. Write your answer to `output/answer.md`.
+  2. The answer MUST be a single Markdown line (no headings, no bullet lists).
+  3. The line MUST NOT start with a preamble phrase.
+  4. The line MUST be 120 characters or fewer.
+
+expected:
+  must_use_tools: [Write]
+  forbidden_tools: [Bash]
+  must_produce_files: [output/answer.md]
+  completion_status: [succeeded]
+
+grading:
+  programmatic:
+    weight: 0.6
+    constraints:
+      # The output must be a single non-empty line — no second non-empty line.
+      - type: file_no_pattern
+        file: output/answer.md
+        pattern: '(?m)^.+\n\n?.+'
+      # Must not contain heading markers.
+      - type: file_no_pattern
+        file: output/answer.md
+        pattern: '^#'
+      # Must not start with common preamble phrases.
+      - type: file_no_pattern
+        file: output/answer.md
+        pattern: '(?i)^(certainly|of course|sure[,!]|great question|happy to help|absolutely)[!,.]'
+      # Must not use bullet / numbered lists.
+      - type: file_no_pattern
+        file: output/answer.md
+        pattern: '(?m)^[-*\d]'
+      # Each line ≤ 120 chars (the single content line).
+      - type: file_line_max_chars
+        file: output/answer.md
+        max: 120
+
+  llm_judge:
+    weight: 0.4
+    rubrics:
+      - name: terseness
+        prompt: |
+          The output should be a single terse line (no preamble, no bullet list,
+          no heading) that correctly states the Pythagorean theorem.
+          Score 10 if the answer is ≤ 2 short sentences, factually correct, and
+          starts directly with the mathematical content (e.g. "In a right triangle…"
+          or "a² + b² = c²…").
+          Deduct points proportionally for verbosity, preamble phrases, or inaccuracy.
+        max_score: 10
+      - name: factual_accuracy
+        prompt: |
+          Does the answer correctly state the Pythagorean theorem
+          (a² + b² = c² for a right triangle)?  Score 10 for correct, 0 for wrong.
+        max_score: 10