diff --git a/cmd/intentctl/main.go b/cmd/intentctl/main.go index 2f1e444..7b8b674 100644 --- a/cmd/intentctl/main.go +++ b/cmd/intentctl/main.go @@ -243,18 +243,11 @@ func sortedKeys(m map[string]string) []string { return keys } -// evalCase 与 intent 包评测集（testdata/corpus.json）同格式。 -type evalCase struct { - Name string `json:"name"` - Input string `json:"input"` - ExpectBranch string `json:"expect_branch"` - ExpectTop string `json:"expect_top"` - ExpectSlots map[string]string `json:"expect_slots"` - ExpectMissingSlots []string `json:"expect_missing_slots"` -} - type evalCorpus struct { - Cases []evalCase `json:"cases"` + Cases []intent.CorpusCase `json:"cases"` + BuiltinSpecs []intent.IntentSpec `json:"builtin_specs,omitempty"` + GuardDomains []string `json:"guard_domains,omitempty"` + ProtectedDomains []string `json:"protected_domains,omitempty"` } func cmdEval(args []string) error { @@ -277,7 +270,23 @@ func cmdEval(args []string) error { if len(corpus.Cases) == 0 { return fmt.Errorf("评测集为空") } - reg, err := buildRegistry(c) + // 语料自声明：builtin_specs 先行注册（如 guard 这类 Go 内置形态的 spec）， + // 域名与命令行取并集——评测集可以自包含复现，不依赖隐式内置。 + var reg *intent.Registry + if len(corpus.BuiltinSpecs) > 0 || len(corpus.GuardDomains) > 0 || len(corpus.ProtectedDomains) > 0 { + if c.root == "" { + return fmt.Errorf("-root 必填") + } + guard := append(append([]string(nil), splitDomains(c.guard)...), corpus.GuardDomains...) + protected := append(append([]string(nil), splitDomains(c.protected)...), corpus.ProtectedDomains...) + b := intent.NewRegistryBuilder(intent.Options{GuardDomains: guard, ProtectedDomains: protected}) + if err := b.Register(corpus.BuiltinSpecs...); err != nil { + return fmt.Errorf("语料 builtin_specs 注册失败：%w", err) + } + reg, err = b.BuildFS(context.Background(), os.DirFS(c.root), ".") + } else { + reg, err = buildRegistry(c) + } if err != nil { return fmt.Errorf("配置校验失败：%w", err) } @@ -286,6 +295,21 @@ func cmdEval(args []string) error { failed := 0 for _, tc := range corpus.Cases { + if len(tc.Turns) > 0 { + // v2 多轮：链式预期逐轮执行，逐轮命令期望断言（I1 WP6）。 + result := intent.RunCorpusCase(context.Background(), snap, tc) + if result.Passed() { + fmt.Printf("PASS %-28s 多轮(%d 轮)\n", tc.Name, len(result.Turns)) + continue + } + failed++ + for _, tr := range result.Turns { + for _, mm := range tr.Mismatches { + fmt.Printf("FAIL %-28s turn %d 输入=%q：%s\n", tc.Name, tr.Index, tr.Input, mm) + } + } + continue + } res, err := rec.Recognize(context.Background(), tc.Input, snap) if err != nil { fmt.Printf("FAIL %-28s 识别错误：%v\n", tc.Name, err) diff --git a/intent/corpus.go b/intent/corpus.go new file mode 100644 index 0000000..f820426 --- /dev/null +++ b/intent/corpus.go @@ -0,0 +1,182 @@ +package intent + +import ( + "context" + "fmt" +) + +// CorpusCase 是评测用例：v1 单轮（input/expect_*）与 v2 多轮（turns）共用。 +// v1 字段在 Turns 为空时生效，并自动等价于一个单 turn 用例（不含命令断言）。 +type CorpusCase struct { + Name string `json:"name"` + Input string `json:"input,omitempty"` + ExpectBranch string `json:"expect_branch,omitempty"` + ExpectTop string `json:"expect_top,omitempty"` + ExpectSlots map[string]string `json:"expect_slots,omitempty"` + ExpectMissingSlots []string `json:"expect_missing_slots,omitempty"` + Turns []CorpusTurn `json:"turns,omitempty"` +} + +// CorpusTurn 是多轮用例中的一轮：输入与逐轮期望（分支/top/命令列表）。 +type CorpusTurn struct { + Input string `json:"input"` + ExpectBranch string `json:"expect_branch,omitempty"` + ExpectTop string `json:"expect_top,omitempty"` + ExpectCommands []CommandExpect `json:"expect_commands,omitempty"` +} + +// CommandExpect 是对一条命令的期望；只有写出的字段参与断言。 +type CommandExpect struct { + Kind string `json:"kind,omitempty"` + ActionKind string `json:"action_kind,omitempty"` + ActionID string `json:"action_id,omitempty"` + Slots map[string]string `json:"slots,omitempty"` + Name string `json:"name,omitempty"` + Value string `json:"value,omitempty"` + Missing []string `json:"missing,omitempty"` + Question string `json:"question,omitempty"` + Candidates []string `json:"candidates,omitempty"` +} + +// TurnResult 一轮的实际结果与差异。 +type TurnResult struct { + Index int + Input string + Branch string + TopID string + Commands []Command + Mismatches []string +} + +// CaseResult 一个用例的全部轮次结果。Passed = 所有轮次无差异。 +type CaseResult struct { + Name string + Turns []TurnResult +} + +// Passed 报告用例是否全部轮次通过。 +func (c CaseResult) Passed() bool { + for _, t := range c.Turns { + if len(t.Mismatches) > 0 { + return false + } + } + return true +} + +// chainedExpectation 模拟宿主的挂起预期链：上一轮 Top 有必填缺失或非法 +// 槽位时，下一轮挂起该意图（与产品「带槽启动后追问」的行为对应）。 +type chainedExpectation struct { + next *PendingExpectation +} + +func (c *chainedExpectation) Pending(ctx context.Context) *PendingExpectation { + return c.next +} + +// RunCorpusCase 执行一个用例的全部轮次（v1 用例自动包装为单轮）。 +// 识别器按轮次构建在链式预期上，保证 SetSlot+Start 这类组合可在评测中到达。 +func RunCorpusCase(ctx context.Context, snap *Snapshot, tc CorpusCase) CaseResult { + turns := tc.Turns + if len(turns) == 0 { + turns = []CorpusTurn{{Input: tc.Input, ExpectBranch: tc.ExpectBranch, ExpectTop: tc.ExpectTop}} + } + exp := &chainedExpectation{} + rec := NewRecognizer(Options{}, exp, nil) + out := CaseResult{Name: tc.Name} + for i, turn := range turns { + tr := TurnResult{Index: i, Input: turn.Input} + res, err := rec.Recognize(ctx, turn.Input, snap) + if err != nil { + tr.Mismatches = append(tr.Mismatches, "识别错误："+err.Error()) + out.Turns = append(out.Turns, tr) + break + } + tr.Branch = string(res.Branch) + if res.Top != nil { + tr.TopID = res.Top.ID + } + tr.Commands = res.Commands + tr.Mismatches = matchTurn(turn, res) + out.Turns = append(out.Turns, tr) + exp.next = nil + if res.Top != nil && (len(res.Top.MissingRequiredSlots) > 0 || len(res.Top.InvalidSlots) > 0) { + exp.next = &PendingExpectation{IntentIDs: []string{res.Top.ID}, Reason: "eval chained"} + } + } + return out +} + +func matchTurn(turn CorpusTurn, res Result) []string { + var mm []string + if turn.ExpectBranch != "" && string(res.Branch) != turn.ExpectBranch { + mm = append(mm, fmt.Sprintf("branch=%s(期望 %s)", res.Branch, turn.ExpectBranch)) + } + if turn.ExpectTop != "" { + got := "" + if res.Top != nil { + got = res.Top.ID + } + if got != turn.ExpectTop { + mm = append(mm, fmt.Sprintf("top=%s(期望 %s)", got, turn.ExpectTop)) + } + } + if turn.ExpectCommands != nil { + mm = append(mm, matchCommands(turn.ExpectCommands, res.Commands)...) + } + return mm +} + +func matchCommands(want []CommandExpect, got []Command) []string { + if len(want) != len(got) { + return []string{fmt.Sprintf("commands 数量=%d(期望 %d)", len(got), len(want))} + } + var mm []string + for i, w := range want { + g := got[i] + if w.Kind != "" && string(g.Kind) != w.Kind { + mm = append(mm, fmt.Sprintf("cmd[%d].kind=%s(期望 %s)", i, g.Kind, w.Kind)) + continue + } + if w.ActionKind != "" && g.Action.Kind != w.ActionKind { + mm = append(mm, fmt.Sprintf("cmd[%d].action.kind=%s(期望 %s)", i, g.Action.Kind, w.ActionKind)) + } + if w.ActionID != "" && g.Action.ID != w.ActionID { + mm = append(mm, fmt.Sprintf("cmd[%d].action.id=%s(期望 %s)", i, g.Action.ID, w.ActionID)) + } + for k, v := range w.Slots { + if g.Slots[k] != v { + mm = append(mm, fmt.Sprintf("cmd[%d].slots[%s]=%q(期望 %q)", i, k, g.Slots[k], v)) + } + } + if w.Name != "" && g.Name != w.Name { + mm = append(mm, fmt.Sprintf("cmd[%d].name=%s(期望 %s)", i, g.Name, w.Name)) + } + if w.Value != "" && g.Value != w.Value { + mm = append(mm, fmt.Sprintf("cmd[%d].value=%s(期望 %s)", i, g.Value, w.Value)) + } + for _, m := range w.Missing { + if !containsString(g.Missing, m) { + mm = append(mm, fmt.Sprintf("cmd[%d].missing 缺 %s", i, m)) + } + } + if w.Question != "" && g.Question != w.Question { + mm = append(mm, fmt.Sprintf("cmd[%d].question=%q(期望 %q)", i, g.Question, w.Question)) + } + for _, c := range w.Candidates { + if !containsString(g.Candidates, c) { + mm = append(mm, fmt.Sprintf("cmd[%d].candidates 缺 %s", i, c)) + } + } + } + return mm +} + +func containsString(list []string, s string) bool { + for _, x := range list { + if x == s { + return true + } + } + return false +} diff --git a/intent/corpus_test.go b/intent/corpus_test.go new file mode 100644 index 0000000..352c227 --- /dev/null +++ b/intent/corpus_test.go @@ -0,0 +1,79 @@ +package intent + +import ( + "context" + "encoding/json" + "os" + "strings" + "testing" +) + +// TestEvalMultiTurnCorpus 是 I1 验收载体：v2 多轮语料含「SetSlot 加 Start」 +// 用例并通过（链式预期把第二轮转进 expectation_hit）。 +func TestEvalMultiTurnCorpus(t *testing.T) { + data, err := os.ReadFile("testdata/corpus_multiturn.json") + if err != nil { + t.Fatal(err) + } + var doc struct { + Cases []CorpusCase `json:"cases"` + } + if err := json.Unmarshal(data, &doc); err != nil { + t.Fatal(err) + } + if len(doc.Cases) == 0 { + t.Fatal("empty corpus") + } + _, snap := buildDemoRegistry(t) + for _, tc := range doc.Cases { + result := RunCorpusCase(context.Background(), snap, tc) + if !result.Passed() { + for _, tr := range result.Turns { + for _, mm := range tr.Mismatches { + t.Errorf("case %s turn %d (%q): %s", tc.Name, tr.Index, tr.Input, mm) + } + } + } + } +} + +// TestCorpusV1WrapsAsSingleTurn：v1 单轮形态经同一执行器等价运行（兼容承诺）。 +func TestCorpusV1WrapsAsSingleTurn(t *testing.T) { + _, snap := buildDemoRegistry(t) + tc := CorpusCase{Name: "v1", Input: "你好呀", ExpectBranch: "matched", ExpectTop: "demo.greet.hello"} + result := RunCorpusCase(context.Background(), snap, tc) + if !result.Passed() { + t.Fatalf("v1 wrap: %+v", result.Turns) + } + if len(result.Turns) != 1 || result.Turns[0].Branch != "matched" { + t.Fatalf("turns = %+v", result.Turns) + } +} + +// TestMatchCommandsFindsMismatches：命令断言的差异定位（kind/action/slot/name）。 +func TestMatchCommandsFindsMismatches(t *testing.T) { + got := []Command{ + {Kind: CommandStart, Action: ActionRef{Kind: ActionHandler, ID: "h"}, Slots: map[string]string{"topic": "电机"}}, + } + mm := matchCommands([]CommandExpect{ + {Kind: "start", ActionKind: ActionHandler, ActionID: "other"}, + }, got) + if len(mm) != 1 || !strings.Contains(mm[0], "action.id") { + t.Fatalf("mm = %v", mm) + } + mm = matchCommands([]CommandExpect{ + {Kind: "start", Slots: map[string]string{"topic": "水泵"}}, + }, got) + if len(mm) != 1 || !strings.Contains(mm[0], "slots") { + t.Fatalf("mm = %v", mm) + } + mm = matchCommands([]CommandExpect{ + {Kind: "start", ActionKind: ActionHandler, ActionID: "h", Slots: map[string]string{"topic": "电机"}}, + }, got) + if len(mm) != 0 { + t.Fatalf("mm = %v", mm) + } + if mm := matchCommands([]CommandExpect{{Kind: "start"}, {Kind: "clarify"}}, got); len(mm) != 1 { + t.Fatalf("count mismatch = %v", mm) + } +} diff --git a/intent/testdata/corpus.json b/intent/testdata/corpus.json index 0d01b9e..6c753a1 100644 --- a/intent/testdata/corpus.json +++ b/intent/testdata/corpus.json @@ -68,5 +68,28 @@ "input": "打结", "expect_branch": "ambiguous" } + ], + "guard_domains": [ + "guard" + ], + "protected_domains": [ + "guard" + ], + "builtin_specs": [ + { + "id": "guard.config_change", + "domain": "guard", + "version": 1, + "priority": 1000, + "matcher": { + "includes_any": [ + "系统配置", + "网关", + "改密码" + ] + }, + "risk": "high", + "handler": "refuse.and_guide" + } ] } \ No newline at end of file diff --git a/intent/testdata/corpus_multiturn.json b/intent/testdata/corpus_multiturn.json new file mode 100644 index 0000000..8824feb --- /dev/null +++ b/intent/testdata/corpus_multiturn.json @@ -0,0 +1,48 @@ +{ + "cases": [ + { + "name": "report_fill_slot_two_turns", + "turns": [ + { + "input": "帮我生成报告", + "expect_branch": "matched", + "expect_top": "report.generate", + "expect_commands": [ + { + "kind": "start", + "action_kind": "handler", + "action_id": "reportHandler" + }, + { + "kind": "clarify", + "missing": [ + "topic" + ], + "question": "要查询哪个主题？" + } + ] + }, + { + "input": "生成报告，主题是电机", + "expect_branch": "expectation_hit", + "expect_top": "report.generate", + "expect_commands": [ + { + "kind": "set_slot", + "name": "topic", + "value": "电机" + }, + { + "kind": "start", + "action_kind": "handler", + "action_id": "reportHandler", + "slots": { + "topic": "电机" + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/intent/testdata/specs/base.json b/intent/testdata/specs/base.json index d651077..78a3711 100644 --- a/intent/testdata/specs/base.json +++ b/intent/testdata/specs/base.json @@ -129,6 +129,26 @@ ] }, "priority": 5 + }, + { + "id": "report.generate", + "domain": "report", + "version": 1, + "matcher": { + "includes_any": [ + "生成报告" + ] + }, + "slots": [ + { + "key": "topic", + "required": true, + "pattern": "主题是(.+)", + "question": "要查询哪个主题？", + "label": "主题" + } + ], + "handler": "reportHandler" } ], "disable": []