Files
Agent-Intent/cmd/intentctl/main.go
T
weichen 1d0fcbd1ba
ci / vet-and-test (pull_request) Successful in 18s
ci / docker-checkout-hostgateway (pull_request) Successful in 0s
Add the multi-turn eval format and self-describing corpora
Why: I1 WP6 - eval needs per-turn command expectations to carry the
SetSlot+Start acceptance case, and corpora must reproduce standalone
in intentctl (the guard case has been red there since before I1
because the guard spec only existed as a test builtin).

Evidence: go vet clean; go test ./... all ok; intentctl eval passes
11/11 on the v1 corpus (guard case fixed via self-declaration) and
1/1 on the new two-turn SetSlot+Start corpus.

Impact: new intent/corpus.go (CorpusCase/CorpusTurn/CommandExpect,
RunCorpusCase with chained expectations); intentctl eval consumes
turns and corpus-declared builtin_specs and domains; report.generate
spec and corpus_multiturn.json fixtures added. v1 corpus format and
conformance unchanged.

Refs: Documents proposal 2026-10-07-agent-stack-n1q1i1-plans.md I1 WP6
2026-10-07 08:12:50 +00:00

348 lines
10 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// intentctl 是意图配置的只读诊断 CLI:lint 校验配置并报告冲突,
// explain 输出单条输入的裁决轨迹,eval 跑评测集。所有命令不改配置、不写文件。
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"os"
"sort"
"strings"
"agent-intent/intent"
)
func main() {
if err := run(os.Args[1:]); err != nil {
fmt.Fprintln(os.Stderr, "intentctl:", err)
os.Exit(1)
}
}
func run(args []string) error {
if len(args) < 1 {
return usage()
}
var err error
switch args[0] {
case "lint":
err = cmdLint(args[1:])
case "explain":
err = cmdExplain(args[1:])
case "eval":
err = cmdEval(args[1:])
case "catalog":
err = cmdCatalog(args[1:])
default:
err = usage()
}
return err
}
func usage() error {
return fmt.Errorf("用法:intentctl <lint|explain|eval|catalog> ...\n lint -root <dir> [-protected a,b] [-guard a,b]\n explain -root <dir> [-protected a,b] [-guard a,b] <input>\n eval -root <dir> [-protected a,b] [-guard a,b] <corpus.json>\n catalog -root <dir> [-protected a,b] [-guard a,b] 输出能力目录(注入规划者/领导提示词用)")
}
type commonOpts struct {
root string
protected string
guard string
}
func parseCommon(fs *flag.FlagSet) *commonOpts {
c := &commonOpts{}
fs.StringVar(&c.root, "root", "", "配置目录(*.json,必填)")
fs.StringVar(&c.protected, "protected", "guard", "保护域,逗号分隔")
fs.StringVar(&c.guard, "guard", "guard", "守卫域,逗号分隔")
return c
}
func buildRegistry(c *commonOpts) (*intent.Registry, error) {
if c.root == "" {
return nil, fmt.Errorf("-root 必填")
}
b := intent.NewRegistryBuilder(intent.Options{
ProtectedDomains: splitDomains(c.protected),
GuardDomains: splitDomains(c.guard),
})
return b.BuildFS(context.Background(), os.DirFS(c.root), ".")
}
func splitDomains(s string) []string {
if strings.TrimSpace(s) == "" {
return nil
}
var out []string
for _, p := range strings.Split(s, ",") {
if p = strings.TrimSpace(p); p != "" {
out = append(out, p)
}
}
return out
}
func cmdLint(args []string) error {
fs := flag.NewFlagSet("lint", flag.ContinueOnError)
c := parseCommon(fs)
if err := fs.Parse(args); err != nil {
return err
}
reg, err := buildRegistry(c)
if err != nil {
return fmt.Errorf("配置校验失败:%w", err)
}
snap := reg.Snapshot()
fmt.Printf("快照 revision=%d 启用=%d 禁用=%d\n", snap.Revision(), len(snap.SpecIDs()), len(snap.DisabledIDs()))
for _, id := range snap.SpecIDs() {
src, _ := snap.Source(id)
fmt.Printf(" spec %s (%s)\n", id, src)
}
for _, id := range snap.DisabledIDs() {
src, _ := snap.Source(id)
fmt.Printf(" disabled %s (%s)\n", id, src)
}
conflicts := snap.Conflicts()
if len(conflicts) == 0 {
fmt.Println("共享词条提示:无")
} else {
fmt.Printf("共享词条提示(%d 条,非错误,计分与确定性次序会裁决):\n", len(conflicts))
for _, cf := range conflicts {
fmt.Printf(" %s 与 %s 共享词条 %q\n", cf.A, cf.B, cf.Term)
}
}
// 词界提示(I3):短词条且未开 boundary 的 spec 容易被中文子串误命中
// (如「配置」命中「配置项」)。提示非错误;boundary: true 可压掉。
var hints []string
for _, id := range snap.SpecIDs() {
spec, ok := snap.SpecByID(id)
if !ok || spec.Matcher.Boundary {
continue
}
terms := append(append([]string(nil), spec.Matcher.IncludesAny...), spec.Matcher.IncludesAll...)
for _, term := range terms {
if runeLen(term) <= 2 {
hints = append(hints, fmt.Sprintf(" %s 词条 %q 短且未开 boundary(中文子串误命中风险;可考虑 boundary: true)", id, term))
}
}
}
if len(hints) == 0 {
fmt.Println("词界提示:无")
return nil
}
fmt.Printf("词界提示(%d 条,非错误):\n", len(hints))
for _, h := range hints {
fmt.Println(h)
}
return nil
}
func runeLen(s string) int {
n := 0
for range s {
n++
}
return n
}
func cmdExplain(args []string) error {
fs := flag.NewFlagSet("explain", flag.ContinueOnError)
c := parseCommon(fs)
if err := fs.Parse(args); err != nil {
return err
}
if fs.NArg() < 1 {
return fmt.Errorf("explain 需要一个输入参数")
}
input := fs.Arg(0)
reg, err := buildRegistry(c)
if err != nil {
return fmt.Errorf("配置校验失败:%w", err)
}
snap := reg.Snapshot()
rec := intent.NewRecognizer(intent.Options{}, nil, nil)
res, err := rec.Recognize(context.Background(), input, snap)
if err != nil {
return err
}
fmt.Printf("输入:%q(%d 字节)\n分支:%s\n", input, len(input), res.Branch)
if res.Top != nil {
fmt.Printf("Top:%s 置信度=%.4f 风险=%s 确认=%v handler=%s\n", res.Top.ID, res.Top.Confidence, res.Top.Risk, res.Top.RequiresConfirm, res.Top.Handler)
if len(res.Top.Slots) > 0 {
keys := make([]string, 0, len(res.Top.Slots))
for k := range res.Top.Slots {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
fmt.Printf(" 槽位 %s=%q\n", k, res.Top.Slots[k])
}
}
if len(res.Top.MissingRequiredSlots) > 0 {
fmt.Printf(" 缺失必填槽位:%s\n", strings.Join(res.Top.MissingRequiredSlots, ","))
}
if len(res.Top.InvalidSlots) > 0 {
fmt.Printf(" 非法槽位:")
for _, k := range sortedKeys(res.Top.InvalidSlots) {
fmt.Printf("%s(%s) ", k, res.Top.InvalidSlots[k])
}
fmt.Println()
}
}
if len(res.Candidates) > 0 {
fmt.Printf("候选(%d):\n", len(res.Candidates))
for _, cand := range res.Candidates {
fmt.Printf(" %.4f %-28s %s\n", cand.Confidence, cand.ID, cand.Reason)
}
}
fmt.Println("裁决轨迹:")
trace := snap.Explain(input)
fmt.Printf(" margin=%.4f\n", trace.AmbiguityMargin)
printStage(" 守卫", trace.Guard, trace.GuardRejected)
if trace.Expectation != nil {
fmt.Printf(" 挂起预期:%v 命中=%s\n", trace.Expectation.PendingIDs, trace.Expectation.HitID)
}
printStage(" 规则", trace.Rules, trace.RulesRejected)
return nil
}
func printStage(name string, scored []intent.ScoredSpec, rejected []intent.RejectedSpec) {
if len(scored) == 0 && len(rejected) == 0 {
fmt.Printf("%s:未评估\n", name)
return
}
for _, s := range scored {
fmt.Printf("%s 命中 %-28s %.4f %s %s\n", name, s.ID, s.Score, strings.Join(s.HitTerms, ","), strings.Join(s.HitPatterns, ","))
}
for _, r := range rejected {
fmt.Printf("%s 排除 %-28s %s\n", name, r.ID, r.Reason)
}
}
func cmdCatalog(args []string) error {
fs := flag.NewFlagSet("catalog", flag.ContinueOnError)
c := parseCommon(fs)
if err := fs.Parse(args); err != nil {
return err
}
reg, err := buildRegistry(c)
if err != nil {
return fmt.Errorf("配置校验失败:%w", err)
}
fmt.Print(reg.Snapshot().CatalogMarkdown())
return nil
}
func sortedKeys(m map[string]string) []string {
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
return keys
}
type evalCorpus struct {
Cases []intent.CorpusCase `json:"cases"`
BuiltinSpecs []intent.IntentSpec `json:"builtin_specs,omitempty"`
GuardDomains []string `json:"guard_domains,omitempty"`
ProtectedDomains []string `json:"protected_domains,omitempty"`
}
func cmdEval(args []string) error {
fs := flag.NewFlagSet("eval", flag.ContinueOnError)
c := parseCommon(fs)
if err := fs.Parse(args); err != nil {
return err
}
if fs.NArg() < 1 {
return fmt.Errorf("eval 需要评测集路径参数")
}
data, err := os.ReadFile(fs.Arg(0))
if err != nil {
return err
}
var corpus evalCorpus
if err := json.Unmarshal(data, &corpus); err != nil {
return fmt.Errorf("评测集解析失败:%w", err)
}
if len(corpus.Cases) == 0 {
return fmt.Errorf("评测集为空")
}
// 语料自声明:builtin_specs 先行注册(如 guard 这类 Go 内置形态的 spec),
// 域名与命令行取并集——评测集可以自包含复现,不依赖隐式内置。
var reg *intent.Registry
if len(corpus.BuiltinSpecs) > 0 || len(corpus.GuardDomains) > 0 || len(corpus.ProtectedDomains) > 0 {
if c.root == "" {
return fmt.Errorf("-root 必填")
}
guard := append(append([]string(nil), splitDomains(c.guard)...), corpus.GuardDomains...)
protected := append(append([]string(nil), splitDomains(c.protected)...), corpus.ProtectedDomains...)
b := intent.NewRegistryBuilder(intent.Options{GuardDomains: guard, ProtectedDomains: protected})
if err := b.Register(corpus.BuiltinSpecs...); err != nil {
return fmt.Errorf("语料 builtin_specs 注册失败:%w", err)
}
reg, err = b.BuildFS(context.Background(), os.DirFS(c.root), ".")
} else {
reg, err = buildRegistry(c)
}
if err != nil {
return fmt.Errorf("配置校验失败:%w", err)
}
snap := reg.Snapshot()
rec := intent.NewRecognizer(intent.Options{}, nil, nil)
failed := 0
for _, tc := range corpus.Cases {
if len(tc.Turns) > 0 {
// v2 多轮:链式预期逐轮执行,逐轮命令期望断言(I1 WP6)。
result := intent.RunCorpusCase(context.Background(), snap, tc)
if result.Passed() {
fmt.Printf("PASS %-28s 多轮(%d 轮)\n", tc.Name, len(result.Turns))
continue
}
failed++
for _, tr := range result.Turns {
for _, mm := range tr.Mismatches {
fmt.Printf("FAIL %-28s turn %d 输入=%q:%s\n", tc.Name, tr.Index, tr.Input, mm)
}
}
continue
}
res, err := rec.Recognize(context.Background(), tc.Input, snap)
if err != nil {
fmt.Printf("FAIL %-28s 识别错误:%v\n", tc.Name, err)
failed++
continue
}
if string(res.Branch) != tc.ExpectBranch || (tc.ExpectTop != "" && (res.Top == nil || res.Top.ID != tc.ExpectTop)) {
gotTop := "(none)"
if res.Top != nil {
gotTop = res.Top.ID
}
fmt.Printf("FAIL %-28s 输入=%q 分支=%s(期望 %s) top=%s(期望 %s)\n", tc.Name, tc.Input, res.Branch, tc.ExpectBranch, gotTop, tc.ExpectTop)
failed++
continue
}
slotOK := true
for k, v := range tc.ExpectSlots {
if res.Top == nil || res.Top.Slots[k] != v {
slotOK = false
break
}
}
if !slotOK {
fmt.Printf("FAIL %-28s 槽位不符:%v\n", tc.Name, res.Top.Slots)
failed++
continue
}
fmt.Printf("PASS %-28s %s\n", tc.Name, res.Branch)
}
fmt.Printf("总计 %d 用例:%d 通过,%d 失败\n", len(corpus.Cases), len(corpus.Cases)-failed, failed)
if failed > 0 {
return fmt.Errorf("%d/%d 用例失败", failed, len(corpus.Cases))
}
return nil
}