weichen/Agent-Intent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add the multi-turn eval format and self-describing corpora (I1 WP6) #8

Merged
weichen merged 1 commits from feature/agent-stack-i1 into main 2026-10-07 08:22:33 +00:00
Conversation 0 Commits 1 Files Changed 6
+388 -12
6 changed files with 388 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/intentctl/main.go
+36 -12
View File
@@ -243,18 +243,11 @@ func sortedKeys(m map[string]string) []string {
return keys
}
// evalCase 与 intent 包评测集（testdata/corpus.json）同格式。
type evalCase struct {
Name string `json:"name"`
Input string `json:"input"`
ExpectBranch string `json:"expect_branch"`
ExpectTop string `json:"expect_top"`
ExpectSlots map[string]string `json:"expect_slots"`
ExpectMissingSlots []string `json:"expect_missing_slots"`
}
type evalCorpus struct {
Cases []evalCase `json:"cases"`
Cases []intent.CorpusCase `json:"cases"`
BuiltinSpecs []intent.IntentSpec `json:"builtin_specs,omitempty"`
GuardDomains []string `json:"guard_domains,omitempty"`
ProtectedDomains []string `json:"protected_domains,omitempty"`
}
func cmdEval(args []string) error {
@@ -277,7 +270,23 @@ func cmdEval(args []string) error {
if len(corpus.Cases) == 0 {
return fmt.Errorf("评测集为空")
}
reg, err := buildRegistry(c)
// 语料自声明：builtin_specs 先行注册（如 guard 这类 Go 内置形态的 spec），
// 域名与命令行取并集——评测集可以自包含复现，不依赖隐式内置。
var reg *intent.Registry
if len(corpus.BuiltinSpecs) > 0 || len(corpus.GuardDomains) > 0 || len(corpus.ProtectedDomains) > 0 {
if c.root == "" {
return fmt.Errorf("-root 必填")
}
guard := append(append([]string(nil), splitDomains(c.guard)...), corpus.GuardDomains...)
protected := append(append([]string(nil), splitDomains(c.protected)...), corpus.ProtectedDomains...)
b := intent.NewRegistryBuilder(intent.Options{GuardDomains: guard, ProtectedDomains: protected})
if err := b.Register(corpus.BuiltinSpecs...); err != nil {
return fmt.Errorf("语料 builtin_specs 注册失败：%w", err)
}
reg, err = b.BuildFS(context.Background(), os.DirFS(c.root), ".")
} else {
reg, err = buildRegistry(c)
}
if err != nil {
return fmt.Errorf("配置校验失败：%w", err)
}
@@ -286,6 +295,21 @@ func cmdEval(args []string) error {
failed := 0
for _, tc := range corpus.Cases {
if len(tc.Turns) > 0 {
// v2 多轮：链式预期逐轮执行，逐轮命令期望断言（I1 WP6）。
result := intent.RunCorpusCase(context.Background(), snap, tc)
if result.Passed() {
fmt.Printf("PASS %-28s 多轮(%d 轮)\n", tc.Name, len(result.Turns))
continue
}
failed++
for _, tr := range result.Turns {
for _, mm := range tr.Mismatches {
fmt.Printf("FAIL %-28s turn %d 输入=%q：%s\n", tc.Name, tr.Index, tr.Input, mm)
}
}
continue
}
res, err := rec.Recognize(context.Background(), tc.Input, snap)
if err != nil {
fmt.Printf("FAIL %-28s 识别错误：%v\n", tc.Name, err)
intent/corpus.go
+182
View File
@@ -0,0 +1,182 @@
package intent
import (
"context"
"fmt"
)
// CorpusCase 是评测用例：v1 单轮（input/expect_*）与 v2 多轮（turns）共用。
// v1 字段在 Turns 为空时生效，并自动等价于一个单 turn 用例（不含命令断言）。
type CorpusCase struct {
Name string `json:"name"`
Input string `json:"input,omitempty"`
ExpectBranch string `json:"expect_branch,omitempty"`
ExpectTop string `json:"expect_top,omitempty"`
ExpectSlots map[string]string `json:"expect_slots,omitempty"`
ExpectMissingSlots []string `json:"expect_missing_slots,omitempty"`
Turns []CorpusTurn `json:"turns,omitempty"`
}
// CorpusTurn 是多轮用例中的一轮：输入与逐轮期望（分支/top/命令列表）。
type CorpusTurn struct {
Input string `json:"input"`
ExpectBranch string `json:"expect_branch,omitempty"`
ExpectTop string `json:"expect_top,omitempty"`
ExpectCommands []CommandExpect `json:"expect_commands,omitempty"`
}
// CommandExpect 是对一条命令的期望；只有写出的字段参与断言。
type CommandExpect struct {
Kind string `json:"kind,omitempty"`
ActionKind string `json:"action_kind,omitempty"`
ActionID string `json:"action_id,omitempty"`
Slots map[string]string `json:"slots,omitempty"`
Name string `json:"name,omitempty"`
Value string `json:"value,omitempty"`
Missing []string `json:"missing,omitempty"`
Question string `json:"question,omitempty"`
Candidates []string `json:"candidates,omitempty"`
}
// TurnResult 一轮的实际结果与差异。
type TurnResult struct {
Index int
Input string
Branch string
TopID string
Commands []Command
Mismatches []string
}
// CaseResult 一个用例的全部轮次结果。Passed = 所有轮次无差异。
type CaseResult struct {
Name string
Turns []TurnResult
}
// Passed 报告用例是否全部轮次通过。
func (c CaseResult) Passed() bool {
for _, t := range c.Turns {
if len(t.Mismatches) > 0 {
return false
}
}
return true
}
// chainedExpectation 模拟宿主的挂起预期链：上一轮 Top 有必填缺失或非法
// 槽位时，下一轮挂起该意图（与产品「带槽启动后追问」的行为对应）。
type chainedExpectation struct {
next *PendingExpectation
}
func (c *chainedExpectation) Pending(ctx context.Context) *PendingExpectation {
return c.next
}
// RunCorpusCase 执行一个用例的全部轮次（v1 用例自动包装为单轮）。
// 识别器按轮次构建在链式预期上，保证 SetSlot+Start 这类组合可在评测中到达。
func RunCorpusCase(ctx context.Context, snap *Snapshot, tc CorpusCase) CaseResult {
turns := tc.Turns
if len(turns) == 0 {
turns = []CorpusTurn{{Input: tc.Input, ExpectBranch: tc.ExpectBranch, ExpectTop: tc.ExpectTop}}
}
exp := &chainedExpectation{}
rec := NewRecognizer(Options{}, exp, nil)
out := CaseResult{Name: tc.Name}
for i, turn := range turns {
tr := TurnResult{Index: i, Input: turn.Input}
res, err := rec.Recognize(ctx, turn.Input, snap)
if err != nil {
tr.Mismatches = append(tr.Mismatches, "识别错误："+err.Error())
out.Turns = append(out.Turns, tr)
break
}
tr.Branch = string(res.Branch)
if res.Top != nil {
tr.TopID = res.Top.ID
}
tr.Commands = res.Commands
tr.Mismatches = matchTurn(turn, res)
out.Turns = append(out.Turns, tr)
exp.next = nil
if res.Top != nil && (len(res.Top.MissingRequiredSlots) > 0 || len(res.Top.InvalidSlots) > 0) {
exp.next = &PendingExpectation{IntentIDs: []string{res.Top.ID}, Reason: "eval chained"}
}
}
return out
}
func matchTurn(turn CorpusTurn, res Result) []string {
var mm []string
if turn.ExpectBranch != "" && string(res.Branch) != turn.ExpectBranch {
mm = append(mm, fmt.Sprintf("branch=%s(期望 %s)", res.Branch, turn.ExpectBranch))
}
if turn.ExpectTop != "" {
got := ""
if res.Top != nil {
got = res.Top.ID
}
if got != turn.ExpectTop {
mm = append(mm, fmt.Sprintf("top=%s(期望 %s)", got, turn.ExpectTop))
}
}
if turn.ExpectCommands != nil {
mm = append(mm, matchCommands(turn.ExpectCommands, res.Commands)...)
}
return mm
}
func matchCommands(want []CommandExpect, got []Command) []string {
if len(want) != len(got) {
return []string{fmt.Sprintf("commands 数量=%d(期望 %d)", len(got), len(want))}
}
var mm []string
for i, w := range want {
g := got[i]
if w.Kind != "" && string(g.Kind) != w.Kind {
mm = append(mm, fmt.Sprintf("cmd[%d].kind=%s(期望 %s)", i, g.Kind, w.Kind))
continue
}
if w.ActionKind != "" && g.Action.Kind != w.ActionKind {
mm = append(mm, fmt.Sprintf("cmd[%d].action.kind=%s(期望 %s)", i, g.Action.Kind, w.ActionKind))
}
if w.ActionID != "" && g.Action.ID != w.ActionID {
mm = append(mm, fmt.Sprintf("cmd[%d].action.id=%s(期望 %s)", i, g.Action.ID, w.ActionID))
}
for k, v := range w.Slots {
if g.Slots[k] != v {
mm = append(mm, fmt.Sprintf("cmd[%d].slots[%s]=%q(期望 %q)", i, k, g.Slots[k], v))
}
}
if w.Name != "" && g.Name != w.Name {
mm = append(mm, fmt.Sprintf("cmd[%d].name=%s(期望 %s)", i, g.Name, w.Name))
}
if w.Value != "" && g.Value != w.Value {
mm = append(mm, fmt.Sprintf("cmd[%d].value=%s(期望 %s)", i, g.Value, w.Value))
}
for _, m := range w.Missing {
if !containsString(g.Missing, m) {
mm = append(mm, fmt.Sprintf("cmd[%d].missing 缺 %s", i, m))
}
}
if w.Question != "" && g.Question != w.Question {
mm = append(mm, fmt.Sprintf("cmd[%d].question=%q(期望 %q)", i, g.Question, w.Question))
}
for _, c := range w.Candidates {
if !containsString(g.Candidates, c) {
mm = append(mm, fmt.Sprintf("cmd[%d].candidates 缺 %s", i, c))
}
}
}
return mm
}
func containsString(list []string, s string) bool {
for _, x := range list {
if x == s {
return true
}
}
return false
}
intent/corpus_test.go
+79
View File
@@ -0,0 +1,79 @@
package intent
import (
"context"
"encoding/json"
"os"
"strings"
"testing"
)
// TestEvalMultiTurnCorpus 是 I1 验收载体：v2 多轮语料含「SetSlot 加 Start」
// 用例并通过（链式预期把第二轮转进 expectation_hit）。
func TestEvalMultiTurnCorpus(t *testing.T) {
data, err := os.ReadFile("testdata/corpus_multiturn.json")
if err != nil {
t.Fatal(err)
}
var doc struct {
Cases []CorpusCase `json:"cases"`
}
if err := json.Unmarshal(data, &doc); err != nil {
t.Fatal(err)
}
if len(doc.Cases) == 0 {
t.Fatal("empty corpus")
}
_, snap := buildDemoRegistry(t)
for _, tc := range doc.Cases {
result := RunCorpusCase(context.Background(), snap, tc)
if !result.Passed() {
for _, tr := range result.Turns {
for _, mm := range tr.Mismatches {
t.Errorf("case %s turn %d (%q): %s", tc.Name, tr.Index, tr.Input, mm)
}
}
}
}
}
// TestCorpusV1WrapsAsSingleTurn：v1 单轮形态经同一执行器等价运行（兼容承诺）。
func TestCorpusV1WrapsAsSingleTurn(t *testing.T) {
_, snap := buildDemoRegistry(t)
tc := CorpusCase{Name: "v1", Input: "你好呀", ExpectBranch: "matched", ExpectTop: "demo.greet.hello"}
result := RunCorpusCase(context.Background(), snap, tc)
if !result.Passed() {
t.Fatalf("v1 wrap: %+v", result.Turns)
}
if len(result.Turns) != 1 || result.Turns[0].Branch != "matched" {
t.Fatalf("turns = %+v", result.Turns)
}
}
// TestMatchCommandsFindsMismatches：命令断言的差异定位（kind/action/slot/name）。
func TestMatchCommandsFindsMismatches(t *testing.T) {
got := []Command{
{Kind: CommandStart, Action: ActionRef{Kind: ActionHandler, ID: "h"}, Slots: map[string]string{"topic": "电机"}},
}
mm := matchCommands([]CommandExpect{
{Kind: "start", ActionKind: ActionHandler, ActionID: "other"},
}, got)
if len(mm) != 1 || !strings.Contains(mm[0], "action.id") {
t.Fatalf("mm = %v", mm)
}
mm = matchCommands([]CommandExpect{
{Kind: "start", Slots: map[string]string{"topic": "水泵"}},
}, got)
if len(mm) != 1 || !strings.Contains(mm[0], "slots") {
t.Fatalf("mm = %v", mm)
}
mm = matchCommands([]CommandExpect{
{Kind: "start", ActionKind: ActionHandler, ActionID: "h", Slots: map[string]string{"topic": "电机"}},
}, got)
if len(mm) != 0 {
t.Fatalf("mm = %v", mm)
}
if mm := matchCommands([]CommandExpect{{Kind: "start"}, {Kind: "clarify"}}, got); len(mm) != 1 {
t.Fatalf("count mismatch = %v", mm)
}
}
intent/testdata/corpus.json
Vendored
+23
View File
@@ -68,5 +68,28 @@
"input": "打结",
"expect_branch": "ambiguous"
}
],
"guard_domains": [
"guard"
],
"protected_domains": [
"guard"
],
"builtin_specs": [
{
"id": "guard.config_change",
"domain": "guard",
"version": 1,
"priority": 1000,
"matcher": {
"includes_any": [
"系统配置",
"网关",
"改密码"
]
},
"risk": "high",
"handler": "refuse.and_guide"
}
]
}
intent/testdata/corpus_multiturn.json
Vendored
+48
View File
@@ -0,0 +1,48 @@
{
"cases": [
{
"name": "report_fill_slot_two_turns",
"turns": [
{
"input": "帮我生成报告",
"expect_branch": "matched",
"expect_top": "report.generate",
"expect_commands": [
{
"kind": "start",
"action_kind": "handler",
"action_id": "reportHandler"
},
{
"kind": "clarify",
"missing": [
"topic"
],
"question": "要查询哪个主题？"
}
]
},
{
"input": "生成报告，主题是电机",
"expect_branch": "expectation_hit",
"expect_top": "report.generate",
"expect_commands": [
{
"kind": "set_slot",
"name": "topic",
"value": "电机"
},
{
"kind": "start",
"action_kind": "handler",
"action_id": "reportHandler",
"slots": {
"topic": "电机"
}
}
]
}
]
}
]
}
intent/testdata/specs/base.json
Vendored
+20
View File
@@ -129,6 +129,26 @@
]
},
"priority": 5
},
{
"id": "report.generate",
"domain": "report",
"version": 1,
"matcher": {
"includes_any": [
"生成报告"
]
},
"slots": [
{
"key": "topic",
"required": true,
"pattern": "主题是(.+)",
"question": "要查询哪个主题？",
"label": "主题"
}
],
"handler": "reportHandler"
}
],
"disable": []