Add the sanitized recall evaluation set and metrics baseline

Why: A3 - the recall decision (retrieval? model tuning?) needs a
repeatable offline baseline; the eval mechanism existed but had no
corpus and no metrics.
Evidence: five-category corpus (30+ cases) derived from fixture specs;
intentctl eval prints per-category metrics and writes metrics.json;
structural coverage test (verify-a3.json, exit=0).
Impact: evaluation corpus/metrics/baseline report + eval metrics
output; intentctl contract backward compatible.
Refs: feature/catalog-recall-eval-a3

cmd/intentctl/main.go

@@ -1,5 +1,6 @@
 // intentctl 是意图配置的只读诊断 CLI：lint 校验配置并报告冲突，
-// explain 输出单条输入的裁决轨迹，eval 跑评测集。所有命令不改配置、不写文件。
+// explain 输出单条输入的裁决轨迹，eval 跑评测集。所有命令不改配置；
+// 唯一的写文件入口是 eval 显式给出的 -metrics-out（指标 JSON）。
 package main
 
 import (
@@ -42,7 +43,7 @@ func run(args []string) error {
 }
 
 func usage() error {
-	return fmt.Errorf("用法：intentctl <lint|explain|eval|catalog> ...\n  lint    -root <dir> [-protected a,b] [-guard a,b]\n  explain -root <dir> [-protected a,b] [-guard a,b] <input>\n  eval    -root <dir> [-protected a,b] [-guard a,b] <corpus.json>\n  catalog -root <dir> [-protected a,b] [-guard a,b]   输出能力目录（注入规划者/领导提示词用）")
+	return fmt.Errorf("用法：intentctl <lint|explain|eval|catalog> ...\n  lint    -root <dir> [-protected a,b] [-guard a,b]\n  explain -root <dir> [-protected a,b] [-guard a,b] <input>\n  eval    -root <dir> [-protected a,b] [-guard a,b] [-metrics-out <file|->] <corpus.json>\n  catalog -root <dir> [-protected a,b] [-guard a,b]   输出能力目录（注入规划者/领导提示词用）")
 }
 
 type commonOpts struct {
@@ -243,16 +244,20 @@ func sortedKeys(m map[string]string) []string {
 	return keys
 }
 
+// evalCorpus 的 record_only 声明语料为基线评测集：期望描述理想行为，
+// 差异照常打印并计入指标，但不使 eval 失败（不设质量阈值）。
 type evalCorpus struct {
-	Cases            []intent.CorpusCase  `json:"cases"`
-	BuiltinSpecs     []intent.IntentSpec  `json:"builtin_specs,omitempty"`
-	GuardDomains     []string             `json:"guard_domains,omitempty"`
-	ProtectedDomains []string             `json:"protected_domains,omitempty"`
+	Cases            []intent.CorpusCase `json:"cases"`
+	BuiltinSpecs     []intent.IntentSpec `json:"builtin_specs,omitempty"`
+	GuardDomains     []string            `json:"guard_domains,omitempty"`
+	ProtectedDomains []string            `json:"protected_domains,omitempty"`
+	RecordOnly       bool                `json:"record_only,omitempty"`
 }
 
 func cmdEval(args []string) error {
 	fs := flag.NewFlagSet("eval", flag.ContinueOnError)
 	c := parseCommon(fs)
+	metricsOut := fs.String("metrics-out", "", "指标 JSON 输出路径（- 为标准输出；留空不输出）")
 	if err := fs.Parse(args); err != nil {
 		return err
 	}
@@ -356,8 +361,36 @@ func cmdEval(args []string) error {
 		rs = fmt.Sprintf("%.3f", r)
 	}
 	fmt.Printf("指标: precision=%s recall=%s（TP=%d FP=%d FN=%d）\n", ps, rs, metrics.TP, metrics.FP, metrics.FN)
+	summary := intent.SummarizeCorpus(intent.CorpusDigest(data), corpus.Cases, results)
+	fmt.Print(summary.Table())
+	if *metricsOut != "" {
+		if err := writeMetrics(*metricsOut, summary); err != nil {
+			return err
+		}
+	}
 	if failed > 0 {
+		if corpus.RecordOnly {
+			fmt.Printf("基线语料（record_only）：%d/%d 用例与理想期望不符，只记录不判失败\n", failed, len(corpus.Cases))
+			return nil
+		}
 		return fmt.Errorf("%d/%d 用例失败", failed, len(corpus.Cases))
 	}
 	return nil
 }
+
+func writeMetrics(path string, summary intent.EvalSummary) error {
+	out, err := json.MarshalIndent(summary, "", "  ")
+	if err != nil {
+		return err
+	}
+	out = append(out, '\n')
+	if path == "-" {
+		_, err = os.Stdout.Write(out)
+		return err
+	}
+	if err := os.WriteFile(path, out, 0o644); err != nil {
+		return fmt.Errorf("写指标文件失败：%w", err)
+	}
+	fmt.Printf("指标 JSON 已写入 %s\n", path)
+	return nil
+}

cmd/intentctl/main_test.go

@@ -1,9 +1,13 @@
 package main
 
 import (
+	"encoding/json"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
+
+	"agent-intent/intent"
 )
 
 const testSpecJSON = `{
@@ -71,3 +75,38 @@ func TestEvalFailsOnWrongExpectation(t *testing.T) {
 		t.Fatal("expected eval failure on wrong expectation")
 	}
 }
+
+func TestEvalRecordOnlyWritesMetrics(t *testing.T) {
+	dir := writeTemp(t, "specs.json", testSpecJSON)
+	tmp := t.TempDir()
+	corpus := filepath.Join(tmp, "corpus.json")
+	doc := `{"record_only": true, "cases": [
+	  {"name": "miss", "category": "negation_reference", "input": "不用查进度了", "expect_branch": "fallback"},
+	  {"name": "hit", "input": "天气不错", "expect_branch": "fallback"}]}`
+	if err := os.WriteFile(corpus, []byte(doc), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	out := filepath.Join(tmp, "metrics.json")
+	if err := run([]string{"eval", "-root", dir, "-protected", "", "-metrics-out", out, corpus}); err != nil {
+		t.Fatalf("record_only eval should not fail: %v", err)
+	}
+	data, err := os.ReadFile(out)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var s intent.EvalSummary
+	if err := json.Unmarshal(data, &s); err != nil {
+		t.Fatal(err)
+	}
+	if s.Schema != intent.EvalMetricsSchema || s.Overall.Cases != 2 || len(s.Categories) != 2 || s.Overall.MisExecutions != 1 {
+		t.Fatalf("summary = %+v", s)
+	}
+
+	strict := filepath.Join(tmp, "strict.json")
+	if err := os.WriteFile(strict, []byte(strings.Replace(doc, `"record_only": true, `, "", 1)), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if err := run([]string{"eval", "-root", dir, "-protected", "", strict}); err == nil {
+		t.Fatal("corpus without record_only must still fail on mismatch")
+	}
+}

evaluation/BASELINE.md

@@ -0,0 +1,85 @@
+# 能力召回评测基线（A3）
+
+本文记录召回评测集在当前识别器上的**指标基线**。只记录，不设质量阈值、不做通过判定；
+据此决定后续方向（检索、模型调优等）不在本基线范围内。机器可读基线为同目录
+[`metrics.json`](metrics.json)（schema `agent-intent.eval-metrics/v1`），本表数字与之一致。
+
+## 语料
+
+- 位置：[`intent/testdata/recall_corpus.json`](../intent/testdata/recall_corpus.json)，
+  与 `intent/testdata/corpus.json`、`corpus_multiturn.json` 同一格式（`CorpusCase`），
+  由 `intentctl eval` 与 `go test` 共用。
+- 摘要：`sha256:53d26cfc4caad994074db7d49924e3eabcba7c8ada4d6389a9a23f3085bc2885`
+  （语料文件原始字节的 sha256；语料变更后须重跑并更新本文与 `metrics.json`）。
+- 数据：全部为合成用例，只复用测试夹具 `intent/testdata/specs/base.json` 的域、动作与槽位形态，
+  以及语料自声明的守卫 spec `guard.config_change`；不含真实用户数据。
+- 期望写的是**理想行为**（不是当前行为的快照），语料声明 `record_only: true`：
+  与期望不符的用例照常打印 FAIL 并计入指标，但不使 eval 失败。
+- 离线：生成路径用确定性回放生成器（回放用例 `generator_commands`），无模型、无网络调用。
+
+### 类别覆盖
+
+| 类别 | 用例 | 轮次 | 覆盖内容 |
+|---|---|---|---|
+| `negation_reference` | 8 | 10 | 否定指令（不要/别/不用/先不要）、`excludes_any` 否定、「还是刚才那个」类指代（单轮无上下文与多轮） |
+| `multi_intent` | 7 | 7 | 一句含两个意图信号：动作+动作（应澄清）、寒暄+动作（应取动作）、动作+守卫（应守卫拦截） |
+| `wrong_slot` | 7 | 7 | 槽位缺失、中文数字、两个候选值、越出字符集、仅符号、空主题 |
+| `expired_confirmation` | 7 | 14 | 上一轮挂起预期（链式或宿主声明）在下一轮到达前过期：裸回答应回落、完整新指令应走规则 matched 而非 expectation_hit；含 1 例未过期对照 |
+| `generation_consistency` | 8 | 8 | 同一输入同时走规则路径与生成路径（回放生成器 + 识别器兜底同款校验），比对 Top 与槽位；含槽位分歧、错动作、幻觉动作、闲聊 |
+| 合计 | 37 | 46 | |
+
+## 指标口径
+
+按轮次计（v1 单轮用例即一轮）；比率保留 4 位，分母为 0 记 `-`（JSON `null`）。
+
+- **pass**：全部轮次与理想期望一致的用例数。
+- **top1**：写了 `expect_top` 的轮次中，实际 Top 等于期望的比例。
+- **mis_exec（误执行）**：实际 `matched`/`expectation_hit` 出了 Top，但与期望动作不同
+  （期望别的 Top，或期望 `fallback`/`ambiguous`/`guard_hit` 却执行了）——危险项。
+- **clarify（澄清率）**：`guard_hit` 或产出 `clarify` 命令的轮次占比。
+- **fallback（回落率）**：实际 `fallback` 的轮次占比。
+- **gen_agree（生成一致率）**：`generation_consistency` 用例中规则路径与生成路径
+  Top ID 与槽位都一致的比例（两边都无 Top 也算一致）。
+
+## 基线（本次运行）
+
+| 类别 | 用例 | pass | 轮次 | top1 | mis_exec | clarify | fallback | gen_agree |
+|---|---|---|---|---|---|---|---|---|
+| `expired_confirmation` | 7 | 7 | 14 | 11/11 = 1.0000 | 0 | 5/14 = 0.3571 | 3/14 = 0.2143 | - |
+| `generation_consistency` | 8 | 8 | 8 | 7/7 = 1.0000 | 0 | 0/8 = 0.0000 | 1/8 = 0.1250 | 5/8 = 0.6250 |
+| `multi_intent` | 7 | 6 | 7 | 2/3 = 0.6667 | 1 | 5/7 = 0.7143 | 0/7 = 0.0000 | - |
+| `negation_reference` | 8 | 3 | 10 | 3/3 = 1.0000 | 5 | 0/10 = 0.0000 | 2/10 = 0.2000 | - |
+| `wrong_slot` | 7 | 3 | 7 | 7/7 = 1.0000 | 0 | 3/7 = 0.4286 | 0/7 = 0.0000 | - |
+| **overall** | 37 | 27 | 46 | 30/31 = 0.9677 | 6 | 13/46 = 0.2826 | 6/46 = 0.1304 | 5/8 = 0.6250 |
+
+同次运行的既有 I2 指标：precision=0.967 recall=0.967（TP=29 FP=1 FN=1）。
+
+### 记录到的差异（事实，不含判定）
+
+- `negation_reference`：5 次误执行均为语义否定被当作肯定——「不要重启任务」「别取消任务 42」
+  「不用查进度了」「先不要发布」「刚才那个任务先别重启」分别 matched 到 restart/cancel/status_query/deploy/restart；
+  只有 `excludes_any` 声明的否定词（「配置」）被拦下。
+- `multi_intent`：「你好，帮我生成报告，主题是电机」按 priority 平分裁决到 `demo.greet.hello`（1 次误执行）；
+  两个动作并列的输入均进入 ambiguous 澄清。
+- `wrong_slot`：两个候选值（「任务 42 和任务 43」「任务 42 还是任务 43」）、越出字符集（`abc_def` 抽成 `abc`）、
+  仅符号（`--`）都直接 start、未澄清；缺失与中文数字正确进入 clarify。
+- `expired_confirmation`：过期后裸回答回落、完整指令走规则 matched，均与期望一致。
+- `generation_consistency`：3 例不一致——生成槽位 `run_id=43` 与规则 `42` 不同、生成错动作
+  `taskrun.restart`（规则为 status_query）、幻觉动作 `ghost.flow` 被校验 fail-closed 后无 Top。
+
+## 重跑
+
+在仓库根（依赖子模块已按 gitlink 初始化）：
+
+```bash
+# 打印逐例 PASS/FAIL、指标表，并写机器基线（-metrics-out - 则输出到标准输出）
+GOWORK=off go run ./cmd/intentctl eval -root intent/testdata/specs \
+  -metrics-out evaluation/metrics.json intent/testdata/recall_corpus.json
+
+# 结构覆盖测试（≥30 例、五类齐全、每例可执行、指标 JSON 合法且与计数一致、基线摘要与语料一致）
+GOWORK=off go test -count=1 -run 'TestRecallCorpusStructure' -v ./intent/
+```
+
+`-root` 指向测试同款夹具目录；守卫 spec 与守卫/保护域由语料 `builtin_specs`、`guard_domains`、
+`protected_domains` 自声明，测试按同样方式建快照，因此两条命令的指标同源。
+识别是确定性的：同语料、同代码重跑得到逐字节相同的 `metrics.json`。

evaluation/metrics.json

@@ -0,0 +1,108 @@
+{
+  "schema": "agent-intent.eval-metrics/v1",
+  "corpus_digest": "sha256:53d26cfc4caad994074db7d49924e3eabcba7c8ada4d6389a9a23f3085bc2885",
+  "overall": {
+    "category": "overall",
+    "cases": 37,
+    "passed_cases": 27,
+    "turns": 46,
+    "top1_expected": 31,
+    "top1_hits": 30,
+    "top1_hit_rate": 0.9677,
+    "mis_executions": 6,
+    "clarifications": 13,
+    "clarification_rate": 0.2826,
+    "fallbacks": 6,
+    "fallback_rate": 0.1304,
+    "generation_cases": 8,
+    "generation_agree": 5,
+    "generation_agreement_rate": 0.625
+  },
+  "categories": [
+    {
+      "category": "expired_confirmation",
+      "cases": 7,
+      "passed_cases": 7,
+      "turns": 14,
+      "top1_expected": 11,
+      "top1_hits": 11,
+      "top1_hit_rate": 1,
+      "mis_executions": 0,
+      "clarifications": 5,
+      "clarification_rate": 0.3571,
+      "fallbacks": 3,
+      "fallback_rate": 0.2143,
+      "generation_cases": 0,
+      "generation_agree": 0,
+      "generation_agreement_rate": null
+    },
+    {
+      "category": "generation_consistency",
+      "cases": 8,
+      "passed_cases": 8,
+      "turns": 8,
+      "top1_expected": 7,
+      "top1_hits": 7,
+      "top1_hit_rate": 1,
+      "mis_executions": 0,
+      "clarifications": 0,
+      "clarification_rate": 0,
+      "fallbacks": 1,
+      "fallback_rate": 0.125,
+      "generation_cases": 8,
+      "generation_agree": 5,
+      "generation_agreement_rate": 0.625
+    },
+    {
+      "category": "multi_intent",
+      "cases": 7,
+      "passed_cases": 6,
+      "turns": 7,
+      "top1_expected": 3,
+      "top1_hits": 2,
+      "top1_hit_rate": 0.6667,
+      "mis_executions": 1,
+      "clarifications": 5,
+      "clarification_rate": 0.7143,
+      "fallbacks": 0,
+      "fallback_rate": 0,
+      "generation_cases": 0,
+      "generation_agree": 0,
+      "generation_agreement_rate": null
+    },
+    {
+      "category": "negation_reference",
+      "cases": 8,
+      "passed_cases": 3,
+      "turns": 10,
+      "top1_expected": 3,
+      "top1_hits": 3,
+      "top1_hit_rate": 1,
+      "mis_executions": 5,
+      "clarifications": 0,
+      "clarification_rate": 0,
+      "fallbacks": 2,
+      "fallback_rate": 0.2,
+      "generation_cases": 0,
+      "generation_agree": 0,
+      "generation_agreement_rate": null
+    },
+    {
+      "category": "wrong_slot",
+      "cases": 7,
+      "passed_cases": 3,
+      "turns": 7,
+      "top1_expected": 7,
+      "top1_hits": 7,
+      "top1_hit_rate": 1,
+      "mis_executions": 0,
+      "clarifications": 3,
+      "clarification_rate": 0.4286,
+      "fallbacks": 0,
+      "fallback_rate": 0,
+      "generation_cases": 0,
+      "generation_agree": 0,
+      "generation_agreement_rate": null
+    }
+  ]
+}

intent/corpus.go

@@ -7,22 +7,33 @@ import (
 
 // CorpusCase 是评测用例：v1 单轮（input/expect_*）与 v2 多轮（turns）共用。
 // v1 字段在 Turns 为空时生效，并自动等价于一个单 turn 用例（不含命令断言）。
+// Category 与 GeneratorCommands 为可选字段（A3 召回评测）：旧语料不写即按
+// 未分类处理；GeneratorCommands 是确定性回放生成器的脚本输出，写了才在
+// 首轮输入上额外跑一遍生成路径并与规则路径比对。
 type CorpusCase struct {
 	Name               string            `json:"name"`
+	Category           string            `json:"category,omitempty"`
 	Input              string            `json:"input,omitempty"`
 	ExpectBranch       string            `json:"expect_branch,omitempty"`
 	ExpectTop          string            `json:"expect_top,omitempty"`
 	ExpectSlots        map[string]string `json:"expect_slots,omitempty"`
 	ExpectMissingSlots []string          `json:"expect_missing_slots,omitempty"`
 	Turns              []CorpusTurn      `json:"turns,omitempty"`
+	GeneratorCommands  []Command         `json:"generator_commands,omitempty"`
 }
 
-// CorpusTurn 是多轮用例中的一轮：输入与逐轮期望（分支/top/命令列表）。
+// CorpusTurn 是多轮用例中的一轮：输入与逐轮期望（分支/top/槽位/命令列表）。
+// PendingExpectation 声明本轮到达前宿主挂起的预期意图（覆盖上一轮的链式预期）；
+// ExpectationExpired 表示本轮到达时挂起预期已过期（声明的或链式的都作废）。
 type CorpusTurn struct {
-	Input          string          `json:"input"`
-	ExpectBranch   string          `json:"expect_branch,omitempty"`
-	ExpectTop      string          `json:"expect_top,omitempty"`
-	ExpectCommands []CommandExpect `json:"expect_commands,omitempty"`
+	Input              string            `json:"input"`
+	PendingExpectation []string          `json:"pending_expectation,omitempty"`
+	ExpectationExpired bool              `json:"expectation_expired,omitempty"`
+	ExpectBranch       string            `json:"expect_branch,omitempty"`
+	ExpectTop          string            `json:"expect_top,omitempty"`
+	ExpectSlots        map[string]string `json:"expect_slots,omitempty"`
+	ExpectMissingSlots []string          `json:"expect_missing_slots,omitempty"`
+	ExpectCommands     []CommandExpect   `json:"expect_commands,omitempty"`
 }
 
 // CommandExpect 是对一条命令的期望；只有写出的字段参与断言。
@@ -44,14 +55,29 @@ type TurnResult struct {
 	Input      string
 	Branch     string
 	TopID      string
+	Slots      map[string]string
 	Commands   []Command
 	Mismatches []string
 }
 
+// GeneratorComparison 是同一输入的规则路径与生成路径（确定性回放生成器，
+// 经与识别器兜底同款的校验）的 Top 比对。Agree = Top ID 与槽位都一致
+// （两边都没有 Top 也算一致）。
+type GeneratorComparison struct {
+	Input          string
+	RulesTopID     string
+	GeneratorTopID string
+	RulesSlots     map[string]string
+	GeneratorSlots map[string]string
+	Agree          bool
+}
+
 // CaseResult 一个用例的全部轮次结果。Passed = 所有轮次无差异。
+// Generator 仅在用例写了 generator_commands 时非 nil；比对结果不计入 Passed。
 type CaseResult struct {
-	Name  string
-	Turns []TurnResult
+	Name      string
+	Turns     []TurnResult
+	Generator *GeneratorComparison
 }
 
 // Passed 报告用例是否全部轮次通过。
@@ -74,17 +100,31 @@ func (c *chainedExpectation) Pending(ctx context.Context) *PendingExpectation {
 	return c.next
 }
 
-// RunCorpusCase 执行一个用例的全部轮次（v1 用例自动包装为单轮）。
-// 识别器按轮次构建在链式预期上，保证 SetSlot+Start 这类组合可在评测中到达。
-func RunCorpusCase(ctx context.Context, snap *Snapshot, tc CorpusCase) CaseResult {
-	turns := tc.Turns
-	if len(turns) == 0 {
-		turns = []CorpusTurn{{Input: tc.Input, ExpectBranch: tc.ExpectBranch, ExpectTop: tc.ExpectTop}}
+// corpusTurns 返回用例的轮次（v1 用例自动包装为单轮，携带 v1 槽位期望）。
+func corpusTurns(tc CorpusCase) []CorpusTurn {
+	if len(tc.Turns) > 0 {
+		return tc.Turns
 	}
+	return []CorpusTurn{{Input: tc.Input, ExpectBranch: tc.ExpectBranch, ExpectTop: tc.ExpectTop,
+		ExpectSlots: tc.ExpectSlots, ExpectMissingSlots: tc.ExpectMissingSlots}}
+}
+
+// RunCorpusCase 执行一个用例的全部轮次（v1 用例自动包装为单轮）。
+// 识别器按轮次构建在链式预期上，保证 SetSlot+Start 这类组合可在评测中到达；
+// 轮次可显式声明挂起预期或声明其已过期。写了 generator_commands 的用例
+// 另在首轮输入上跑生成路径并与规则路径比对（离线、确定性回放）。
+func RunCorpusCase(ctx context.Context, snap *Snapshot, tc CorpusCase) CaseResult {
+	turns := corpusTurns(tc)
 	exp := &chainedExpectation{}
 	rec := NewRecognizer(Options{}, exp, nil)
 	out := CaseResult{Name: tc.Name}
 	for i, turn := range turns {
+		if len(turn.PendingExpectation) > 0 {
+			exp.next = &PendingExpectation{IntentIDs: append([]string(nil), turn.PendingExpectation...), Reason: "eval declared"}
+		}
+		if turn.ExpectationExpired {
+			exp.next = nil
+		}
 		tr := TurnResult{Index: i, Input: turn.Input}
 		res, err := rec.Recognize(ctx, turn.Input, snap)
 		if err != nil {
@@ -95,6 +135,7 @@ func RunCorpusCase(ctx context.Context, snap *Snapshot, tc CorpusCase) CaseResul
 		tr.Branch = string(res.Branch)
 		if res.Top != nil {
 			tr.TopID = res.Top.ID
+			tr.Slots = res.Top.Slots
 		}
 		tr.Commands = res.Commands
 		tr.Mismatches = matchTurn(turn, res)
@@ -104,9 +145,52 @@ func RunCorpusCase(ctx context.Context, snap *Snapshot, tc CorpusCase) CaseResul
 			exp.next = &PendingExpectation{IntentIDs: []string{res.Top.ID}, Reason: "eval chained"}
 		}
 	}
+	if len(tc.GeneratorCommands) > 0 && len(out.Turns) > 0 {
+		out.Generator = compareGeneratorPath(ctx, snap, out.Turns[0], tc.GeneratorCommands)
+	}
 	return out
 }
 
+// replayGenerator 是评测用确定性生成器：忽略输入与目录，回放用例脚本中的命令
+// （无模型、无网络）。
+type replayGenerator struct {
+	cmds []Command
+}
+
+func (g replayGenerator) Generate(ctx context.Context, input string, catalog []CatalogSpec) ([]Command, error) {
+	out := make([]Command, len(g.cmds))
+	for i, c := range g.cmds {
+		c.Slots = cloneStringMap(c.Slots)
+		out[i] = c
+	}
+	return out, nil
+}
+
+// compareGeneratorPath 绕过规则阶段，直接走识别器兜底同款的生成路径
+// （目录 top-k → 生成 → 校验 → 解析回权威 spec），再与规则路径首轮比对。
+func compareGeneratorPath(ctx context.Context, snap *Snapshot, rules TurnResult, cmds []Command) *GeneratorComparison {
+	cmp := &GeneratorComparison{Input: rules.Input, RulesTopID: rules.TopID, RulesSlots: rules.Slots}
+	rec := NewRecognizer(Options{Generator: replayGenerator{cmds: cmds}}, nil, nil)
+	if res, ok := rec.generateCommands(ctx, rules.Input, snap); ok && res.Top != nil {
+		cmp.GeneratorTopID = res.Top.ID
+		cmp.GeneratorSlots = res.Top.Slots
+	}
+	cmp.Agree = cmp.RulesTopID == cmp.GeneratorTopID && sameSlots(cmp.RulesSlots, cmp.GeneratorSlots)
+	return cmp
+}
+
+func sameSlots(a, b map[string]string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for k, v := range a {
+		if w, ok := b[k]; !ok || w != v {
+			return false
+		}
+	}
+	return true
+}
+
 func matchTurn(turn CorpusTurn, res Result) []string {
 	var mm []string
 	if turn.ExpectBranch != "" && string(res.Branch) != turn.ExpectBranch {
@@ -121,6 +205,24 @@ func matchTurn(turn CorpusTurn, res Result) []string {
 			mm = append(mm, fmt.Sprintf("top=%s(期望 %s)", got, turn.ExpectTop))
 		}
 	}
+	for _, k := range sortedKeys(turn.ExpectSlots) {
+		got := ""
+		if res.Top != nil {
+			got = res.Top.Slots[k]
+		}
+		if got != turn.ExpectSlots[k] {
+			mm = append(mm, fmt.Sprintf("slots[%s]=%q(期望 %q)", k, got, turn.ExpectSlots[k]))
+		}
+	}
+	if len(turn.ExpectMissingSlots) > 0 {
+		var got []string
+		if res.Top != nil {
+			got = res.Top.MissingRequiredSlots
+		}
+		if fmt.Sprint(got) != fmt.Sprint(turn.ExpectMissingSlots) {
+			mm = append(mm, fmt.Sprintf("missing=%v(期望 %v)", got, turn.ExpectMissingSlots))
+		}
+	}
 	if turn.ExpectCommands != nil {
 		mm = append(mm, matchCommands(turn.ExpectCommands, res.Commands)...)
 	}
@@ -208,6 +310,7 @@ func (m CorpusMetrics) Recall() (float64, bool) {
 //   - FP：实际 matched/top=Y 但期望的标签不是 matched/top=Y
 //     （期望别的 top、期望 fallback/ambiguous、或期望 matched 但 top 不同）；
 //   - FN：期望 matched/top=X 但实际没拿到它（branch 不对或 top 不同）。
+//
 // 只有写了 expect_top 的轮次进入指标（意图识别通行口径）。
 func MetricsOf(cases []CorpusCase, results []CaseResult) CorpusMetrics {
 	var m CorpusMetrics

intent/eval_test.go

@@ -75,4 +75,16 @@ func TestEvalCorpus(t *testing.T) {
 	_, snap := buildDemoRegistry(t)
 	rec := NewRecognizer(Options{}, nil, nil)
 	runCorpus(t, rec, snap, doc.Cases)
+
+	var shared struct {
+		Cases []CorpusCase `json:"cases"`
+	}
+	if err := json.Unmarshal(data, &shared); err != nil {
+		t.Fatal(err)
+	}
+	var results []CaseResult
+	for _, tc := range shared.Cases {
+		results = append(results, RunCorpusCase(context.Background(), snap, tc))
+	}
+	t.Log("\n" + SummarizeCorpus(CorpusDigest(data), shared.Cases, results).Table())
 }

intent/evalmetrics.go

@@ -0,0 +1,193 @@
+package intent
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"math"
+	"sort"
+	"strings"
+)
+
+// EvalMetricsSchema 是机器可读评测指标（EvalSummary JSON）的格式标识。
+const EvalMetricsSchema = "agent-intent.eval-metrics/v1"
+
+// UncategorizedCategory 是未写 category 的用例归入的类别（旧语料兼容）。
+const UncategorizedCategory = "uncategorized"
+
+// CategoryMetrics 是一个类别（或全量）的评测指标（A3 召回基线）。
+// 计数口径按轮次（v1 用例即一轮），比率的分母为 0 时为 nil（JSON null）：
+//   - top1：写了 expect_top 的轮次中实际 Top 等于期望的比例；
+//   - mis_executions：实际 matched/expectation_hit 出 Top，但与期望动作不同
+//     （期望别的 top，或期望 fallback/ambiguous/guard_hit 却执行了）——危险项；
+//   - clarification：guard_hit 或产出 clarify 命令的轮次；
+//   - fallback：实际 fallback 的轮次；
+//   - generation：写了 generator_commands 的用例中，规则路径与生成路径一致的比例。
+type CategoryMetrics struct {
+	Category                string   `json:"category"`
+	Cases                   int      `json:"cases"`
+	PassedCases             int      `json:"passed_cases"`
+	Turns                   int      `json:"turns"`
+	Top1Expected            int      `json:"top1_expected"`
+	Top1Hits                int      `json:"top1_hits"`
+	Top1HitRate             *float64 `json:"top1_hit_rate"`
+	MisExecutions           int      `json:"mis_executions"`
+	Clarifications          int      `json:"clarifications"`
+	ClarificationRate       *float64 `json:"clarification_rate"`
+	Fallbacks               int      `json:"fallbacks"`
+	FallbackRate            *float64 `json:"fallback_rate"`
+	GenerationCases         int      `json:"generation_cases"`
+	GenerationAgree         int      `json:"generation_agree"`
+	GenerationAgreementRate *float64 `json:"generation_agreement_rate"`
+}
+
+// EvalSummary 是一次评测的指标汇总：按类别（名称升序）与全量，附语料摘要。
+// 只记录基线，不含阈值与通过判定。
+type EvalSummary struct {
+	Schema       string            `json:"schema"`
+	CorpusDigest string            `json:"corpus_digest"`
+	Overall      CategoryMetrics   `json:"overall"`
+	Categories   []CategoryMetrics `json:"categories"`
+}
+
+// CorpusDigest 返回语料文件原始字节的摘要（sha256:<hex>）。
+func CorpusDigest(data []byte) string {
+	sum := sha256.Sum256(data)
+	return "sha256:" + hex.EncodeToString(sum[:])
+}
+
+// SummarizeCorpus 汇总评测指标。results 须与 cases 按下标一一对应
+// （即 RunCorpusCase 逐个执行 cases 的结果）。
+func SummarizeCorpus(digest string, cases []CorpusCase, results []CaseResult) EvalSummary {
+	byCat := map[string]*CategoryMetrics{}
+	overall := &CategoryMetrics{Category: "overall"}
+	for i, tc := range cases {
+		if i >= len(results) {
+			break
+		}
+		cat := tc.Category
+		if strings.TrimSpace(cat) == "" {
+			cat = UncategorizedCategory
+		}
+		m, ok := byCat[cat]
+		if !ok {
+			m = &CategoryMetrics{Category: cat}
+			byCat[cat] = m
+		}
+		accumulateCase(m, tc, results[i])
+		accumulateCase(overall, tc, results[i])
+	}
+	out := EvalSummary{Schema: EvalMetricsSchema, CorpusDigest: digest, Categories: []CategoryMetrics{}}
+	for _, m := range byCat {
+		finishRates(m)
+		out.Categories = append(out.Categories, *m)
+	}
+	sort.Slice(out.Categories, func(i, j int) bool { return out.Categories[i].Category < out.Categories[j].Category })
+	finishRates(overall)
+	out.Overall = *overall
+	return out
+}
+
+func accumulateCase(m *CategoryMetrics, tc CorpusCase, res CaseResult) {
+	m.Cases++
+	if res.Passed() {
+		m.PassedCases++
+	}
+	turns := corpusTurns(tc)
+	for i, tr := range res.Turns {
+		if i >= len(turns) {
+			break
+		}
+		want := turns[i]
+		m.Turns++
+		if want.ExpectTop != "" {
+			m.Top1Expected++
+			if tr.TopID == want.ExpectTop {
+				m.Top1Hits++
+			}
+		}
+		if misExecuted(want, tr) {
+			m.MisExecutions++
+		}
+		if tr.Branch == string(BranchGuardHit) || hasClarify(tr.Commands) {
+			m.Clarifications++
+		}
+		if tr.Branch == string(BranchFallback) {
+			m.Fallbacks++
+		}
+	}
+	if res.Generator != nil {
+		m.GenerationCases++
+		if res.Generator.Agree {
+			m.GenerationAgree++
+		}
+	}
+}
+
+// misExecuted 报告该轮是否执行了与期望不同的动作（危险项）。
+// 期望 matched/expectation_hit 但未写 expect_top 的轮次不约束动作，不计。
+func misExecuted(want CorpusTurn, got TurnResult) bool {
+	executed := got.TopID != "" && (got.Branch == string(BranchMatched) || got.Branch == string(BranchExpectationHit))
+	if !executed {
+		return false
+	}
+	if want.ExpectTop != "" {
+		return got.TopID != want.ExpectTop
+	}
+	switch Branch(want.ExpectBranch) {
+	case BranchFallback, BranchAmbiguous, BranchGuardHit:
+		return true
+	}
+	return false
+}
+
+func hasClarify(cmds []Command) bool {
+	for _, c := range cmds {
+		if c.Kind == CommandClarify {
+			return true
+		}
+	}
+	return false
+}
+
+func finishRates(m *CategoryMetrics) {
+	m.Top1HitRate = ratio(m.Top1Hits, m.Top1Expected)
+	m.ClarificationRate = ratio(m.Clarifications, m.Turns)
+	m.FallbackRate = ratio(m.Fallbacks, m.Turns)
+	m.GenerationAgreementRate = ratio(m.GenerationAgree, m.GenerationCases)
+}
+
+// ratio 定点 4 位舍入，保证重跑 JSON 字节稳定；分母为 0 返回 nil。
+func ratio(n, d int) *float64 {
+	if d == 0 {
+		return nil
+	}
+	v := math.Round(float64(n)/float64(d)*10000) / 10000
+	return &v
+}
+
+// Table 返回人类可读的指标表（每类别一行 + overall 行）。
+func (s EvalSummary) Table() string {
+	var b strings.Builder
+	fmt.Fprintf(&b, "指标表（corpus %s）：\n", s.CorpusDigest)
+	fmt.Fprintf(&b, "  %-24s %5s %5s %5s %-14s %8s %-14s %-14s %s\n",
+		"category", "cases", "pass", "turns", "top1", "mis_exec", "clarify", "fallback", "gen_agree")
+	rows := append(append([]CategoryMetrics(nil), s.Categories...), s.Overall)
+	for _, m := range rows {
+		fmt.Fprintf(&b, "  %-24s %5d %5d %5d %-14s %8d %-14s %-14s %s\n",
+			m.Category, m.Cases, m.PassedCases, m.Turns,
+			rateCell(m.Top1Hits, m.Top1Expected, m.Top1HitRate),
+			m.MisExecutions,
+			rateCell(m.Clarifications, m.Turns, m.ClarificationRate),
+			rateCell(m.Fallbacks, m.Turns, m.FallbackRate),
+			rateCell(m.GenerationAgree, m.GenerationCases, m.GenerationAgreementRate))
+	}
+	return b.String()
+}
+
+func rateCell(n, d int, r *float64) string {
+	if r == nil {
+		return "-"
+	}
+	return fmt.Sprintf("%d/%d=%.3f", n, d, *r)
+}

intent/recall_eval_test.go

@@ -0,0 +1,188 @@
+package intent
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"os"
+	"strings"
+	"testing"
+)
+
+// recallCorpusDoc 与 intentctl eval 读取的语料结构一致（自声明守卫 spec 与域）。
+type recallCorpusDoc struct {
+	RecordOnly       bool         `json:"record_only"`
+	Cases            []CorpusCase `json:"cases"`
+	BuiltinSpecs     []IntentSpec `json:"builtin_specs"`
+	GuardDomains     []string     `json:"guard_domains"`
+	ProtectedDomains []string     `json:"protected_domains"`
+}
+
+var recallCategories = []string{
+	"expired_confirmation",
+	"generation_consistency",
+	"multi_intent",
+	"negation_reference",
+	"wrong_slot",
+}
+
+const (
+	recallCorpusPath   = "testdata/recall_corpus.json"
+	recallBaselinePath = "../evaluation/metrics.json"
+)
+
+// runRecallCorpus 按 intentctl eval 同款方式建快照（语料 builtin_specs + 夹具 specs）
+// 并逐例执行，保证测试与 CLI 的指标同源。
+func runRecallCorpus(t *testing.T) ([]byte, recallCorpusDoc, []CaseResult) {
+	t.Helper()
+	data, err := os.ReadFile(recallCorpusPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var doc recallCorpusDoc
+	if err := json.Unmarshal(data, &doc); err != nil {
+		t.Fatal(err)
+	}
+	b := NewRegistryBuilder(Options{GuardDomains: doc.GuardDomains, ProtectedDomains: doc.ProtectedDomains})
+	if err := b.Register(doc.BuiltinSpecs...); err != nil {
+		t.Fatal(err)
+	}
+	reg, err := b.BuildFS(context.Background(), os.DirFS("testdata"), "specs")
+	if err != nil {
+		t.Fatal(err)
+	}
+	snap := reg.Snapshot()
+	results := make([]CaseResult, 0, len(doc.Cases))
+	for _, tc := range doc.Cases {
+		results = append(results, RunCorpusCase(context.Background(), snap, tc))
+	}
+	return data, doc, results
+}
+
+// TestRecallCorpusStructure 只断言结构覆盖（A3）：≥30 例、五类齐全、每例可执行、
+// 指标 JSON 合法且与用例计数一致；不断言任何质量数值（基线只记录）。
+func TestRecallCorpusStructure(t *testing.T) {
+	data, doc, results := runRecallCorpus(t)
+	if !doc.RecordOnly {
+		t.Fatal("召回基线语料须声明 record_only")
+	}
+	if len(doc.Cases) < 30 {
+		t.Fatalf("cases = %d, want >= 30", len(doc.Cases))
+	}
+	perCat := map[string]int{}
+	for i, tc := range doc.Cases {
+		perCat[tc.Category]++
+		turns := corpusTurns(tc)
+		res := results[i]
+		if len(res.Turns) != len(turns) {
+			t.Fatalf("case %s executed %d/%d turns", tc.Name, len(res.Turns), len(turns))
+		}
+		for _, tr := range res.Turns {
+			for _, mm := range tr.Mismatches {
+				if strings.HasPrefix(mm, "识别错误") {
+					t.Fatalf("case %s turn %d: %s", tc.Name, tr.Index, mm)
+				}
+			}
+		}
+		if tc.Category == "generation_consistency" {
+			if len(tc.GeneratorCommands) == 0 || len(turns) != 1 || res.Generator == nil {
+				t.Fatalf("case %s: generation_consistency 须为带 generator_commands 的单轮用例", tc.Name)
+			}
+		}
+	}
+	for _, cat := range recallCategories {
+		if perCat[cat] == 0 {
+			t.Fatalf("category %s missing (have %v)", cat, perCat)
+		}
+	}
+	if len(perCat) != len(recallCategories) {
+		t.Fatalf("unexpected categories %v, want exactly %v", perCat, recallCategories)
+	}
+
+	summary := SummarizeCorpus(CorpusDigest(data), doc.Cases, results)
+	t.Log("\n" + summary.Table())
+	raw, err := json.Marshal(summary)
+	if err != nil {
+		t.Fatal(err)
+	}
+	checkSummaryJSON(t, "fresh", raw, CorpusDigest(data), doc.Cases)
+
+	baseline, err := os.ReadFile(recallBaselinePath)
+	if err != nil {
+		t.Fatalf("read committed baseline: %v", err)
+	}
+	checkSummaryJSON(t, "baseline", baseline, CorpusDigest(data), doc.Cases)
+}
+
+// checkSummaryJSON 校验指标 JSON 结构与计数一致性（不比较质量数值）。
+// 已提交基线的摘要必须等于当前语料摘要：改语料后须按 evaluation/BASELINE.md 重跑。
+func checkSummaryJSON(t *testing.T, label string, raw []byte, digest string, cases []CorpusCase) {
+	t.Helper()
+	dec := json.NewDecoder(bytes.NewReader(raw))
+	dec.DisallowUnknownFields()
+	var s EvalSummary
+	if err := dec.Decode(&s); err != nil {
+		t.Fatalf("%s: metrics JSON malformed: %v", label, err)
+	}
+	if s.Schema != EvalMetricsSchema {
+		t.Fatalf("%s: schema = %q", label, s.Schema)
+	}
+	if s.CorpusDigest != digest {
+		t.Fatalf("%s: corpus_digest = %s, want %s（语料已变更，需重跑基线）", label, s.CorpusDigest, digest)
+	}
+	wantCases, wantTurns, wantGen := map[string]int{}, map[string]int{}, map[string]int{}
+	totalTurns, totalGen := 0, 0
+	for _, tc := range cases {
+		n := len(corpusTurns(tc))
+		wantCases[tc.Category]++
+		wantTurns[tc.Category] += n
+		totalTurns += n
+		if len(tc.GeneratorCommands) > 0 {
+			wantGen[tc.Category]++
+			totalGen++
+		}
+	}
+	if len(s.Categories) != len(wantCases) {
+		t.Fatalf("%s: categories = %d, want %d", label, len(s.Categories), len(wantCases))
+	}
+	sumCases, sumTurns := 0, 0
+	for _, m := range s.Categories {
+		if m.Cases != wantCases[m.Category] || m.Turns != wantTurns[m.Category] || m.GenerationCases != wantGen[m.Category] {
+			t.Fatalf("%s: %s counts = %+v, want cases=%d turns=%d gen=%d", label, m.Category, m, wantCases[m.Category], wantTurns[m.Category], wantGen[m.Category])
+		}
+		checkMetricsBounds(t, label, m)
+		sumCases += m.Cases
+		sumTurns += m.Turns
+	}
+	o := s.Overall
+	if o.Cases != len(cases) || sumCases != o.Cases || o.Turns != totalTurns || sumTurns != o.Turns || o.GenerationCases != totalGen {
+		t.Fatalf("%s: overall = %+v, want cases=%d turns=%d gen=%d", label, o, len(cases), totalTurns, totalGen)
+	}
+	checkMetricsBounds(t, label, o)
+}
+
+func checkMetricsBounds(t *testing.T, label string, m CategoryMetrics) {
+	t.Helper()
+	pairs := []struct {
+		name string
+		n, d int
+		rate *float64
+	}{
+		{"top1", m.Top1Hits, m.Top1Expected, m.Top1HitRate},
+		{"clarification", m.Clarifications, m.Turns, m.ClarificationRate},
+		{"fallback", m.Fallbacks, m.Turns, m.FallbackRate},
+		{"generation", m.GenerationAgree, m.GenerationCases, m.GenerationAgreementRate},
+	}
+	for _, p := range pairs {
+		if p.n < 0 || p.n > p.d {
+			t.Fatalf("%s: %s %s = %d/%d out of range", label, m.Category, p.name, p.n, p.d)
+		}
+		want := ratio(p.n, p.d)
+		if (want == nil) != (p.rate == nil) || (want != nil && *want != *p.rate) {
+			t.Fatalf("%s: %s %s rate inconsistent with %d/%d", label, m.Category, p.name, p.n, p.d)
+		}
+	}
+	if m.PassedCases > m.Cases || m.MisExecutions > m.Turns || m.Top1Expected > m.Turns {
+		t.Fatalf("%s: %s counts out of range: %+v", label, m.Category, m)
+	}
+}

intent/testdata/recall_corpus.json

@@ -0,0 +1,575 @@
+{
+  "record_only": true,
+  "cases": [
+    {
+      "name": "neg_do_not_restart",
+      "category": "negation_reference",
+      "input": "不要重启任务",
+      "expect_branch": "fallback"
+    },
+    {
+      "name": "neg_do_not_cancel_run",
+      "category": "negation_reference",
+      "input": "别取消任务 42",
+      "expect_branch": "fallback"
+    },
+    {
+      "name": "neg_no_need_progress",
+      "category": "negation_reference",
+      "input": "不用查进度了",
+      "expect_branch": "fallback"
+    },
+    {
+      "name": "neg_do_not_deploy",
+      "category": "negation_reference",
+      "input": "先不要发布",
+      "expect_branch": "fallback"
+    },
+    {
+      "name": "neg_excludes_config_progress",
+      "category": "negation_reference",
+      "input": "进度配置在哪里看",
+      "expect_branch": "fallback"
+    },
+    {
+      "name": "ref_same_as_before_no_context",
+      "category": "negation_reference",
+      "input": "还是刚才那个",
+      "expect_branch": "fallback"
+    },
+    {
+      "name": "ref_same_after_status",
+      "category": "negation_reference",
+      "turns": [
+        {
+          "input": "查询任务 42 的进度",
+          "expect_branch": "matched",
+          "expect_top": "demo.run.status_query",
+          "expect_slots": {
+            "run_id": "42"
+          }
+        },
+        {
+          "input": "还是刚才那个，再看下进度",
+          "expect_branch": "matched",
+          "expect_top": "demo.run.status_query"
+        }
+      ]
+    },
+    {
+      "name": "ref_previous_task_hold_restart",
+      "category": "negation_reference",
+      "turns": [
+        {
+          "input": "查询任务 42 的进度",
+          "expect_branch": "matched",
+          "expect_top": "demo.run.status_query"
+        },
+        {
+          "input": "刚才那个任务先别重启",
+          "expect_branch": "fallback"
+        }
+      ]
+    },
+    {
+      "name": "multi_status_then_restart",
+      "category": "multi_intent",
+      "input": "查询任务 42 的进度，然后重启任务",
+      "expect_branch": "ambiguous"
+    },
+    {
+      "name": "multi_greet_then_report",
+      "category": "multi_intent",
+      "input": "你好，帮我生成报告，主题是电机",
+      "expect_branch": "matched",
+      "expect_top": "report.generate",
+      "expect_slots": {
+        "topic": "电机"
+      }
+    },
+    {
+      "name": "multi_cancel_and_deploy",
+      "category": "multi_intent",
+      "input": "取消任务 42 并发布",
+      "expect_branch": "ambiguous"
+    },
+    {
+      "name": "multi_greet_then_status",
+      "category": "multi_intent",
+      "input": "你好，查一下任务 7 的进度",
+      "expect_branch": "matched",
+      "expect_top": "demo.run.status_query",
+      "expect_slots": {
+        "run_id": "7"
+      }
+    },
+    {
+      "name": "multi_restart_and_deploy",
+      "category": "multi_intent",
+      "input": "重启任务后再发布",
+      "expect_branch": "ambiguous"
+    },
+    {
+      "name": "multi_status_and_guard",
+      "category": "multi_intent",
+      "input": "查一下进度，顺便改密码",
+      "expect_branch": "guard_hit",
+      "expect_top": "guard.config_change"
+    },
+    {
+      "name": "multi_report_and_cancel",
+      "category": "multi_intent",
+      "input": "生成报告，主题是电机，再取消任务 9",
+      "expect_branch": "ambiguous"
+    },
+    {
+      "name": "slot_cancel_no_id",
+      "category": "wrong_slot",
+      "turns": [
+        {
+          "input": "取消任务",
+          "expect_branch": "matched",
+          "expect_top": "demo.run.cancel",
+          "expect_missing_slots": [
+            "run_id"
+          ],
+          "expect_commands": [
+            {
+              "kind": "start"
+            },
+            {
+              "kind": "clarify",
+              "missing": [
+                "run_id"
+              ]
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "name": "slot_cancel_chinese_numeral",
+      "category": "wrong_slot",
+      "turns": [
+        {
+          "input": "取消任务四十二",
+          "expect_branch": "matched",
+          "expect_top": "demo.run.cancel",
+          "expect_missing_slots": [
+            "run_id"
+          ],
+          "expect_commands": [
+            {
+              "kind": "start"
+            },
+            {
+              "kind": "clarify",
+              "missing": [
+                "run_id"
+              ]
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "name": "slot_cancel_two_ids",
+      "category": "wrong_slot",
+      "turns": [
+        {
+          "input": "取消任务 42 和任务 43",
+          "expect_branch": "matched",
+          "expect_top": "demo.run.cancel",
+          "expect_commands": [
+            {
+              "kind": "start"
+            },
+            {
+              "kind": "clarify"
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "name": "slot_status_two_ids",
+      "category": "wrong_slot",
+      "turns": [
+        {
+          "input": "查询任务 42 还是任务 43 的进度",
+          "expect_branch": "matched",
+          "expect_top": "demo.run.status_query",
+          "expect_commands": [
+            {
+              "kind": "start"
+            },
+            {
+              "kind": "clarify"
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "name": "slot_status_id_underscore",
+      "category": "wrong_slot",
+      "turns": [
+        {
+          "input": "查询任务 abc_def 的进度",
+          "expect_branch": "matched",
+          "expect_top": "demo.run.status_query",
+          "expect_commands": [
+            {
+              "kind": "start"
+            },
+            {
+              "kind": "clarify"
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "name": "slot_cancel_dash_only",
+      "category": "wrong_slot",
+      "turns": [
+        {
+          "input": "取消任务 --",
+          "expect_branch": "matched",
+          "expect_top": "demo.run.cancel",
+          "expect_commands": [
+            {
+              "kind": "start"
+            },
+            {
+              "kind": "clarify"
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "name": "slot_report_empty_topic",
+      "category": "wrong_slot",
+      "turns": [
+        {
+          "input": "生成报告，主题是",
+          "expect_branch": "matched",
+          "expect_top": "report.generate",
+          "expect_missing_slots": [
+            "topic"
+          ],
+          "expect_commands": [
+            {
+              "kind": "start"
+            },
+            {
+              "kind": "clarify",
+              "missing": [
+                "topic"
+              ],
+              "question": "要查询哪个主题？"
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "name": "exp_cancel_live_control",
+      "category": "expired_confirmation",
+      "turns": [
+        {
+          "input": "取消一下",
+          "expect_branch": "matched",
+          "expect_top": "demo.run.cancel",
+          "expect_missing_slots": [
+            "run_id"
+          ]
+        },
+        {
+          "input": "取消任务 42",
+          "expect_branch": "expectation_hit",
+          "expect_top": "demo.run.cancel",
+          "expect_slots": {
+            "run_id": "42"
+          }
+        }
+      ]
+    },
+    {
+      "name": "exp_cancel_expired_full_command",
+      "category": "expired_confirmation",
+      "turns": [
+        {
+          "input": "取消一下",
+          "expect_branch": "matched",
+          "expect_top": "demo.run.cancel"
+        },
+        {
+          "input": "取消任务 42",
+          "expectation_expired": true,
+          "expect_branch": "matched",
+          "expect_top": "demo.run.cancel",
+          "expect_slots": {
+            "run_id": "42"
+          }
+        }
+      ]
+    },
+    {
+      "name": "exp_cancel_expired_bare_answer",
+      "category": "expired_confirmation",
+      "turns": [
+        {
+          "input": "取消一下",
+          "expect_branch": "matched",
+          "expect_top": "demo.run.cancel"
+        },
+        {
+          "input": "就是任务 42",
+          "expectation_expired": true,
+          "expect_branch": "fallback"
+        }
+      ]
+    },
+    {
+      "name": "exp_report_expired_bare_topic",
+      "category": "expired_confirmation",
+      "turns": [
+        {
+          "input": "帮我生成报告",
+          "expect_branch": "matched",
+          "expect_top": "report.generate"
+        },
+        {
+          "input": "主题是电机",
+          "expectation_expired": true,
+          "expect_branch": "fallback"
+        }
+      ]
+    },
+    {
+      "name": "exp_report_expired_full_command",
+      "category": "expired_confirmation",
+      "turns": [
+        {
+          "input": "帮我生成报告",
+          "expect_branch": "matched",
+          "expect_top": "report.generate"
+        },
+        {
+          "input": "生成报告，主题是水泵",
+          "expectation_expired": true,
+          "expect_branch": "matched",
+          "expect_top": "report.generate",
+          "expect_slots": {
+            "topic": "水泵"
+          }
+        }
+      ]
+    },
+    {
+      "name": "exp_restart_confirm_expired",
+      "category": "expired_confirmation",
+      "turns": [
+        {
+          "input": "把任务重启一下",
+          "expect_branch": "matched",
+          "expect_top": "demo.run.restart"
+        },
+        {
+          "input": "好的，确认",
+          "pending_expectation": [
+            "demo.run.restart"
+          ],
+          "expectation_expired": true,
+          "expect_branch": "fallback"
+        }
+      ]
+    },
+    {
+      "name": "exp_restart_confirm_expired_restated",
+      "category": "expired_confirmation",
+      "turns": [
+        {
+          "input": "把任务重启一下",
+          "expect_branch": "matched",
+          "expect_top": "demo.run.restart"
+        },
+        {
+          "input": "确认重启任务",
+          "pending_expectation": [
+            "demo.run.restart"
+          ],
+          "expectation_expired": true,
+          "expect_branch": "matched",
+          "expect_top": "demo.run.restart"
+        }
+      ]
+    },
+    {
+      "name": "gen_status_agree",
+      "category": "generation_consistency",
+      "input": "查询任务 42 的进度",
+      "expect_branch": "matched",
+      "expect_top": "demo.run.status_query",
+      "expect_slots": {
+        "run_id": "42"
+      },
+      "generator_commands": [
+        {
+          "kind": "start",
+          "action": {
+            "kind": "handler",
+            "id": "taskrun.query_status"
+          },
+          "slots": {
+            "run_id": "42"
+          }
+        }
+      ]
+    },
+    {
+      "name": "gen_restart_agree",
+      "category": "generation_consistency",
+      "input": "把任务重启一下",
+      "expect_branch": "matched",
+      "expect_top": "demo.run.restart",
+      "generator_commands": [
+        {
+          "kind": "start",
+          "action": {
+            "kind": "auto",
+            "id": "demo.run.restart"
+          }
+        }
+      ]
+    },
+    {
+      "name": "gen_report_agree",
+      "category": "generation_consistency",
+      "input": "生成报告，主题是电机",
+      "expect_branch": "matched",
+      "expect_top": "report.generate",
+      "expect_slots": {
+        "topic": "电机"
+      },
+      "generator_commands": [
+        {
+          "kind": "start",
+          "action": {
+            "kind": "handler",
+            "id": "reportHandler"
+          },
+          "slots": {
+            "topic": "电机"
+          }
+        }
+      ]
+    },
+    {
+      "name": "gen_deploy_agree",
+      "category": "generation_consistency",
+      "input": "please DEPLOY it",
+      "expect_branch": "matched",
+      "expect_top": "demo.misc.deploy",
+      "generator_commands": [
+        {
+          "kind": "start",
+          "action": {
+            "kind": "handler",
+            "id": "deploy.run"
+          }
+        }
+      ]
+    },
+    {
+      "name": "gen_cancel_slot_disagree",
+      "category": "generation_consistency",
+      "input": "取消任务 42",
+      "expect_branch": "matched",
+      "expect_top": "demo.run.cancel",
+      "expect_slots": {
+        "run_id": "42"
+      },
+      "generator_commands": [
+        {
+          "kind": "start",
+          "action": {
+            "kind": "auto",
+            "id": "demo.run.cancel"
+          },
+          "slots": {
+            "run_id": "43"
+          }
+        }
+      ]
+    },
+    {
+      "name": "gen_progress_wrong_action",
+      "category": "generation_consistency",
+      "input": "帮我看下进度",
+      "expect_branch": "matched",
+      "expect_top": "demo.run.status_query",
+      "generator_commands": [
+        {
+          "kind": "start",
+          "action": {
+            "kind": "handler",
+            "id": "taskrun.restart"
+          }
+        }
+      ]
+    },
+    {
+      "name": "gen_greet_hallucinated_action",
+      "category": "generation_consistency",
+      "input": "你好呀",
+      "expect_branch": "matched",
+      "expect_top": "demo.greet.hello",
+      "generator_commands": [
+        {
+          "kind": "start",
+          "action": {
+            "kind": "workflow",
+            "id": "ghost.flow"
+          }
+        }
+      ]
+    },
+    {
+      "name": "gen_chitchat_agree",
+      "category": "generation_consistency",
+      "input": "今天天气不错",
+      "expect_branch": "fallback",
+      "generator_commands": [
+        {
+          "kind": "chitchat"
+        }
+      ]
+    }
+  ],
+  "guard_domains": [
+    "guard"
+  ],
+  "protected_domains": [
+    "guard"
+  ],
+  "builtin_specs": [
+    {
+      "id": "guard.config_change",
+      "domain": "guard",
+      "version": 1,
+      "priority": 1000,
+      "matcher": {
+        "includes_any": [
+          "系统配置",
+          "网关",
+          "改密码"
+        ]
+      },
+      "risk": "high",
+      "handler": "refuse.and_guide"
+    }
+  ]
+}