package intent import ( "context" "fmt" ) // CorpusCase 是评测用例:v1 单轮(input/expect_*)与 v2 多轮(turns)共用。 // v1 字段在 Turns 为空时生效,并自动等价于一个单 turn 用例(不含命令断言)。 type CorpusCase struct { Name string `json:"name"` Input string `json:"input,omitempty"` ExpectBranch string `json:"expect_branch,omitempty"` ExpectTop string `json:"expect_top,omitempty"` ExpectSlots map[string]string `json:"expect_slots,omitempty"` ExpectMissingSlots []string `json:"expect_missing_slots,omitempty"` Turns []CorpusTurn `json:"turns,omitempty"` } // CorpusTurn 是多轮用例中的一轮:输入与逐轮期望(分支/top/命令列表)。 type CorpusTurn struct { Input string `json:"input"` ExpectBranch string `json:"expect_branch,omitempty"` ExpectTop string `json:"expect_top,omitempty"` ExpectCommands []CommandExpect `json:"expect_commands,omitempty"` } // CommandExpect 是对一条命令的期望;只有写出的字段参与断言。 type CommandExpect struct { Kind string `json:"kind,omitempty"` ActionKind string `json:"action_kind,omitempty"` ActionID string `json:"action_id,omitempty"` Slots map[string]string `json:"slots,omitempty"` Name string `json:"name,omitempty"` Value string `json:"value,omitempty"` Missing []string `json:"missing,omitempty"` Question string `json:"question,omitempty"` Candidates []string `json:"candidates,omitempty"` } // TurnResult 一轮的实际结果与差异。 type TurnResult struct { Index int Input string Branch string TopID string Commands []Command Mismatches []string } // CaseResult 一个用例的全部轮次结果。Passed = 所有轮次无差异。 type CaseResult struct { Name string Turns []TurnResult } // Passed 报告用例是否全部轮次通过。 func (c CaseResult) Passed() bool { for _, t := range c.Turns { if len(t.Mismatches) > 0 { return false } } return true } // chainedExpectation 模拟宿主的挂起预期链:上一轮 Top 有必填缺失或非法 // 槽位时,下一轮挂起该意图(与产品「带槽启动后追问」的行为对应)。 type chainedExpectation struct { next *PendingExpectation } func (c *chainedExpectation) Pending(ctx context.Context) *PendingExpectation { return c.next } // RunCorpusCase 执行一个用例的全部轮次(v1 用例自动包装为单轮)。 // 识别器按轮次构建在链式预期上,保证 SetSlot+Start 这类组合可在评测中到达。 func RunCorpusCase(ctx context.Context, snap *Snapshot, tc CorpusCase) CaseResult { turns := tc.Turns if len(turns) == 0 { turns = []CorpusTurn{{Input: tc.Input, ExpectBranch: tc.ExpectBranch, ExpectTop: tc.ExpectTop}} } exp := &chainedExpectation{} rec := NewRecognizer(Options{}, exp, nil) out := CaseResult{Name: tc.Name} for i, turn := range turns { tr := TurnResult{Index: i, Input: turn.Input} res, err := rec.Recognize(ctx, turn.Input, snap) if err != nil { tr.Mismatches = append(tr.Mismatches, "识别错误:"+err.Error()) out.Turns = append(out.Turns, tr) break } tr.Branch = string(res.Branch) if res.Top != nil { tr.TopID = res.Top.ID } tr.Commands = res.Commands tr.Mismatches = matchTurn(turn, res) out.Turns = append(out.Turns, tr) exp.next = nil if res.Top != nil && (len(res.Top.MissingRequiredSlots) > 0 || len(res.Top.InvalidSlots) > 0) { exp.next = &PendingExpectation{IntentIDs: []string{res.Top.ID}, Reason: "eval chained"} } } return out } func matchTurn(turn CorpusTurn, res Result) []string { var mm []string if turn.ExpectBranch != "" && string(res.Branch) != turn.ExpectBranch { mm = append(mm, fmt.Sprintf("branch=%s(期望 %s)", res.Branch, turn.ExpectBranch)) } if turn.ExpectTop != "" { got := "" if res.Top != nil { got = res.Top.ID } if got != turn.ExpectTop { mm = append(mm, fmt.Sprintf("top=%s(期望 %s)", got, turn.ExpectTop)) } } if turn.ExpectCommands != nil { mm = append(mm, matchCommands(turn.ExpectCommands, res.Commands)...) } return mm } func matchCommands(want []CommandExpect, got []Command) []string { if len(want) != len(got) { return []string{fmt.Sprintf("commands 数量=%d(期望 %d)", len(got), len(want))} } var mm []string for i, w := range want { g := got[i] if w.Kind != "" && string(g.Kind) != w.Kind { mm = append(mm, fmt.Sprintf("cmd[%d].kind=%s(期望 %s)", i, g.Kind, w.Kind)) continue } if w.ActionKind != "" && g.Action.Kind != w.ActionKind { mm = append(mm, fmt.Sprintf("cmd[%d].action.kind=%s(期望 %s)", i, g.Action.Kind, w.ActionKind)) } if w.ActionID != "" && g.Action.ID != w.ActionID { mm = append(mm, fmt.Sprintf("cmd[%d].action.id=%s(期望 %s)", i, g.Action.ID, w.ActionID)) } for k, v := range w.Slots { if g.Slots[k] != v { mm = append(mm, fmt.Sprintf("cmd[%d].slots[%s]=%q(期望 %q)", i, k, g.Slots[k], v)) } } if w.Name != "" && g.Name != w.Name { mm = append(mm, fmt.Sprintf("cmd[%d].name=%s(期望 %s)", i, g.Name, w.Name)) } if w.Value != "" && g.Value != w.Value { mm = append(mm, fmt.Sprintf("cmd[%d].value=%s(期望 %s)", i, g.Value, w.Value)) } for _, m := range w.Missing { if !containsString(g.Missing, m) { mm = append(mm, fmt.Sprintf("cmd[%d].missing 缺 %s", i, m)) } } if w.Question != "" && g.Question != w.Question { mm = append(mm, fmt.Sprintf("cmd[%d].question=%q(期望 %q)", i, g.Question, w.Question)) } for _, c := range w.Candidates { if !containsString(g.Candidates, c) { mm = append(mm, fmt.Sprintf("cmd[%d].candidates 缺 %s", i, c)) } } } return mm } func containsString(list []string, s string) bool { for _, x := range list { if x == s { return true } } return false } // CorpusMetrics 评测指标(I2):按意图标签(matched/top)计 TP/FP/FN。 // 期望非 matched(fallback/ambiguous)的轮次作负例、不进指标。 type CorpusMetrics struct { TP, FP, FN int } // Precision 精确率;分母为 0 时返回 (0, false)(未定义,由调用方决定展示)。 func (m CorpusMetrics) Precision() (float64, bool) { if m.TP+m.FP == 0 { return 0, false } return float64(m.TP) / float64(m.TP+m.FP), true } // Recall 召回率;分母为 0 时返回 (0, false)。 func (m CorpusMetrics) Recall() (float64, bool) { if m.TP+m.FN == 0 { return 0, false } return float64(m.TP) / float64(m.TP+m.FN), true } // MetricsOf 逐轮累计指标(I2): // - TP:期望 matched/top=X 且实际 matched/top=X; // - FP:实际 matched/top=Y 但期望的标签不是 matched/top=Y // (期望别的 top、期望 fallback/ambiguous、或期望 matched 但 top 不同); // - FN:期望 matched/top=X 但实际没拿到它(branch 不对或 top 不同)。 // 只有写了 expect_top 的轮次进入指标(意图识别通行口径)。 func MetricsOf(cases []CorpusCase, results []CaseResult) CorpusMetrics { var m CorpusMetrics byName := map[string]CorpusCase{} for _, tc := range cases { byName[tc.Name] = tc } for _, res := range results { tc, ok := byName[res.Name] if !ok { continue } turns := tc.Turns if len(turns) == 0 { turns = []CorpusTurn{{ExpectBranch: tc.ExpectBranch, ExpectTop: tc.ExpectTop}} } for i, turn := range turns { if turn.ExpectTop == "" || i >= len(res.Turns) { continue } got := res.Turns[i] wantMatched := turn.ExpectBranch == "" || turn.ExpectBranch == "matched" || turn.ExpectBranch == "expectation_hit" gotMatched := got.Branch == "matched" || got.Branch == "expectation_hit" if gotMatched && got.TopID == turn.ExpectTop && wantMatched { m.TP++ continue } if gotMatched && got.TopID != "" { // 错配同时是假阳(拿了错的)与假阴(期望的没拿到)。 m.FP++ if wantMatched { m.FN++ } continue } if wantMatched { m.FN++ } } } return m }