Merge pull request 'Add precision and recall metrics to the multi-turn evaluation (I2)' (#10) from feature/agent-stack-i2 into main

2026-10-07 11:05:58 +00:00
parent fcf9ec4b2a ab55b54e9b
commit 343b1e6d9a
4 changed files with 324 additions and 0 deletions
@@ -340,6 +340,22 @@ func cmdEval(args []string) error {
 		fmt.Printf("PASS %-28s %s\n", tc.Name, res.Branch)
 	}
 	fmt.Printf("总计 %d 用例：%d 通过，%d 失败\n", len(corpus.Cases), len(corpus.Cases)-failed, failed)
+	// I2：指标输出（precision/recall 按意图标签计；确定性重跑，不计入通过判定）。
+	var results []intent.CaseResult
+	for _, tc := range corpus.Cases {
+		results = append(results, intent.RunCorpusCase(context.Background(), snap, tc))
+	}
+	metrics := intent.MetricsOf(corpus.Cases, results)
+	p, pok := metrics.Precision()
+	r, rok := metrics.Recall()
+	ps, rs := "-", "-"
+	if pok {
+		ps = fmt.Sprintf("%.3f", p)
+	}
+	if rok {
+		rs = fmt.Sprintf("%.3f", r)
+	}
+	fmt.Printf("指标: precision=%s recall=%s（TP=%d FP=%d FN=%d）\n", ps, rs, metrics.TP, metrics.FP, metrics.FN)
 	if failed > 0 {
 		return fmt.Errorf("%d/%d 用例失败", failed, len(corpus.Cases))
 	}
@@ -180,3 +180,73 @@ func containsString(list []string, s string) bool {
 	}
 	return false
 }
+
+// CorpusMetrics 评测指标（I2）：按意图标签（matched/top）计 TP/FP/FN。
+// 期望非 matched（fallback/ambiguous）的轮次作负例、不进指标。
+type CorpusMetrics struct {
+	TP, FP, FN int
+}
+
+// Precision 精确率；分母为 0 时返回 (0, false)（未定义，由调用方决定展示）。
+func (m CorpusMetrics) Precision() (float64, bool) {
+	if m.TP+m.FP == 0 {
+		return 0, false
+	}
+	return float64(m.TP) / float64(m.TP+m.FP), true
+}
+
+// Recall 召回率；分母为 0 时返回 (0, false)。
+func (m CorpusMetrics) Recall() (float64, bool) {
+	if m.TP+m.FN == 0 {
+		return 0, false
+	}
+	return float64(m.TP) / float64(m.TP+m.FN), true
+}
+
+// MetricsOf 逐轮累计指标（I2）：
+//   - TP：期望 matched/top=X 且实际 matched/top=X；
+//   - FP：实际 matched/top=Y 但期望的标签不是 matched/top=Y
+//     （期望别的 top、期望 fallback/ambiguous、或期望 matched 但 top 不同）；
+//   - FN：期望 matched/top=X 但实际没拿到它（branch 不对或 top 不同）。
+// 只有写了 expect_top 的轮次进入指标（意图识别通行口径）。
+func MetricsOf(cases []CorpusCase, results []CaseResult) CorpusMetrics {
+	var m CorpusMetrics
+	byName := map[string]CorpusCase{}
+	for _, tc := range cases {
+		byName[tc.Name] = tc
+	}
+	for _, res := range results {
+		tc, ok := byName[res.Name]
+		if !ok {
+			continue
+		}
+		turns := tc.Turns
+		if len(turns) == 0 {
+			turns = []CorpusTurn{{ExpectBranch: tc.ExpectBranch, ExpectTop: tc.ExpectTop}}
+		}
+		for i, turn := range turns {
+			if turn.ExpectTop == "" || i >= len(res.Turns) {
+				continue
+			}
+			got := res.Turns[i]
+			wantMatched := turn.ExpectBranch == "" || turn.ExpectBranch == "matched" || turn.ExpectBranch == "expectation_hit"
+			gotMatched := got.Branch == "matched" || got.Branch == "expectation_hit"
+			if gotMatched && got.TopID == turn.ExpectTop && wantMatched {
+				m.TP++
+				continue
+			}
+			if gotMatched && got.TopID != "" {
+				// 错配同时是假阳（拿了错的）与假阴（期望的没拿到）。
+				m.FP++
+				if wantMatched {
+					m.FN++
+				}
+				continue
+			}
+			if wantMatched {
+				m.FN++
+			}
+		}
+	}
+	return m
+}
@@ -77,3 +77,51 @@ func TestMatchCommandsFindsMismatches(t *testing.T) {
 		t.Fatalf("count mismatch = %v", mm)
 	}
 }
+
+// TestMetricsOfCounts：指标按意图标签计 TP/FP/FN（I2 验收的指标形态）。
+func TestMetricsOfCounts(t *testing.T) {
+	_, snap := buildDemoRegistry(t)
+	data, err := os.ReadFile("testdata/corpus_multiturn.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var doc struct {
+		Cases []CorpusCase `json:"cases"`
+	}
+	if err := json.Unmarshal(data, &doc); err != nil {
+		t.Fatal(err)
+	}
+	var results []CaseResult
+	for _, tc := range doc.Cases {
+		results = append(results, RunCorpusCase(context.Background(), snap, tc))
+	}
+	m := MetricsOf(doc.Cases, results)
+	// 通过的夹具：指标应当全 TP（precision=recall=1）。
+	if m.FP != 0 || m.FN != 0 {
+		t.Fatalf("metrics = %+v", m)
+	}
+	p, pok := m.Precision()
+	r, rok := m.Recall()
+	if !pok || !rok || p != 1.0 || r != 1.0 {
+		t.Fatalf("precision=%v recall=%v (m=%+v)", p, r, m)
+	}
+	if m.TP < 8 {
+		t.Fatalf("TP = %d, want >= 8（夹具内带 top 期望的轮次）", m.TP)
+	}
+
+	// 手工构造一个 FP+FN 用例验证计数方向。
+	bad := []CorpusCase{{Name: "bad", Turns: []CorpusTurn{
+		{Input: "查询任务 42 的进度", ExpectBranch: "matched", ExpectTop: "demo.greet.hello"},
+	}}}
+	got := []CaseResult{RunCorpusCase(context.Background(), snap, bad[0])}
+	m2 := MetricsOf(bad, got)
+	// 实际 matched status_query 但期望 greet.hello → 1 FP + 1 FN。
+	if m2.TP != 0 || m2.FP != 1 || m2.FN != 1 {
+		t.Fatalf("bad metrics = %+v", m2)
+	}
+	p2, _ := m2.Precision()
+	r2, _ := m2.Recall()
+	if p2 != 0 || r2 != 0 {
+		t.Fatalf("p=%v r=%v, want 0/0", p2, r2)
+	}
+}
@@ -43,6 +43,196 @@
          ]
        }
      ]
+    },
+    {
+      "name": "ambiguous_then_tiebreak_match",
+      "turns": [
+        {
+          "input": "打结",
+          "expect_branch": "ambiguous",
+          "expect_commands": [
+            {
+              "kind": "clarify",
+              "candidates": [
+                "demo.tie.alpha",
+                "demo.tie.beta"
+              ]
+            }
+          ]
+        },
+        {
+          "input": "你好呀",
+          "expect_branch": "matched",
+          "expect_top": "demo.greet.hello"
+        }
+      ]
+    },
+    {
+      "name": "single_matched_full_slots",
+      "turns": [
+        {
+          "input": "查询任务 42 的进度",
+          "expect_branch": "matched",
+          "expect_top": "demo.run.status_query",
+          "expect_commands": [
+            {
+              "kind": "start",
+              "action_kind": "handler",
+              "action_id": "taskrun.query_status",
+              "slots": {
+                "run_id": "42"
+              }
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "name": "single_fallback",
+      "turns": [
+        {
+          "input": "天气不错",
+          "expect_branch": "fallback",
+          "expect_commands": []
+        }
+      ]
+    },
+    {
+      "name": "single_ambiguous_tie",
+      "turns": [
+        {
+          "input": "打结",
+          "expect_branch": "ambiguous",
+          "expect_commands": [
+            {
+              "kind": "clarify",
+              "candidates": [
+                "demo.tie.alpha",
+                "demo.tie.beta"
+              ]
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "name": "single_tiebreak_priority",
+      "turns": [
+        {
+          "input": "你好呀",
+          "expect_branch": "matched",
+          "expect_top": "demo.greet.hello"
+        }
+      ]
+    },
+    {
+      "name": "single_matched_handler_action",
+      "turns": [
+        {
+          "input": "查询任务 7 的进度",
+          "expect_branch": "matched",
+          "expect_top": "demo.run.status_query",
+          "expect_commands": [
+            {
+              "kind": "start",
+              "action_kind": "handler",
+              "action_id": "taskrun.query_status"
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "name": "two_turn_complete_then_fallback",
+      "turns": [
+        {
+          "input": "生成报告，主题是电机",
+          "expect_branch": "matched",
+          "expect_top": "report.generate",
+          "expect_commands": [
+            {
+              "kind": "start",
+              "action_kind": "handler",
+              "action_id": "reportHandler",
+              "slots": {
+                "topic": "电机"
+              }
+            }
+          ]
+        },
+        {
+          "input": "天气不错",
+          "expect_branch": "fallback",
+          "expect_commands": []
+        }
+      ]
+    },
+    {
+      "name": "two_turn_missing_then_other_intent",
+      "turns": [
+        {
+          "input": "帮我生成报告",
+          "expect_branch": "matched",
+          "expect_top": "report.generate"
+        },
+        {
+          "input": "查询任务 42 的进度",
+          "expect_branch": "matched",
+          "expect_top": "demo.run.status_query"
+        }
+      ]
+    },
+    {
+      "name": "single_clarify_missing_with_question",
+      "turns": [
+        {
+          "input": "帮我生成报告",
+          "expect_branch": "matched",
+          "expect_top": "report.generate",
+          "expect_commands": [
+            {
+              "kind": "start"
+            },
+            {
+              "kind": "clarify",
+              "missing": [
+                "topic"
+              ],
+              "question": "要查询哪个主题？"
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "name": "two_turn_set_slot_pump",
+      "turns": [
+        {
+          "input": "帮我生成报告",
+          "expect_branch": "matched",
+          "expect_top": "report.generate"
+        },
+        {
+          "input": "生成报告，主题是水泵",
+          "expect_branch": "expectation_hit",
+          "expect_top": "report.generate",
+          "expect_commands": [
+            {
+              "kind": "set_slot",
+              "name": "topic",
+              "value": "水泵"
+            },
+            {
+              "kind": "start",
+              "action_kind": "handler",
+              "action_id": "reportHandler",
+              "slots": {
+                "topic": "水泵"
+              }
+            }
+          ]
+        }
+      ]
    }
  ]
 }