Author SHA1 Message Date
weichen 6aa4925301 Add the sanitized recall evaluation set and metrics baseline
Why: A3 - the recall decision (retrieval? model tuning?) needs a
repeatable offline baseline; the eval mechanism existed but had no
corpus and no metrics.
Evidence: five-category corpus (30+ cases) derived from fixture specs;
intentctl eval prints per-category metrics and writes metrics.json;
structural coverage test (verify-a3.json, exit=0).
Impact: evaluation corpus/metrics/baseline report + eval metrics
output; intentctl contract backward compatible.
Refs: feature/catalog-recall-eval-a3
2026-10-23 01:02:13 +00:00
weichen 765a7e7177 Isolate snapshots and validate generated commands
Why: F07/F08 - registry and catalog accessors shared caller-mutable
nested state, and the generation fallback skipped full slot validation,
risk metadata and mis-keyed handler slot tables.
Evidence: intent package tests pass incl. new isolation/generated cases
(verify-u5.json).
Impact: no exported signature change; rule and generated paths now share
authoritative validation.
Refs: feature/agent-stack-audit-remediation-b1 U5
2026-10-21 15:41:31 +00:00
16 changed files with 2033 additions and 107 deletions
+39 -6
View File
@@ -1,5 +1,6 @@
// intentctl 是意图配置的只读诊断 CLI:lint 校验配置并报告冲突,
// explain 输出单条输入的裁决轨迹,eval 跑评测集。所有命令不改配置、不写文件。
// explain 输出单条输入的裁决轨迹,eval 跑评测集。所有命令不改配置;
// 唯一的写文件入口是 eval 显式给出的 -metrics-out(指标 JSON)。
package main
import (
@@ -42,7 +43,7 @@ func run(args []string) error {
}
func usage() error {
return fmt.Errorf("用法:intentctl <lint|explain|eval|catalog> ...\n lint -root <dir> [-protected a,b] [-guard a,b]\n explain -root <dir> [-protected a,b] [-guard a,b] <input>\n eval -root <dir> [-protected a,b] [-guard a,b] <corpus.json>\n catalog -root <dir> [-protected a,b] [-guard a,b] 输出能力目录(注入规划者/领导提示词用)")
return fmt.Errorf("用法:intentctl <lint|explain|eval|catalog> ...\n lint -root <dir> [-protected a,b] [-guard a,b]\n explain -root <dir> [-protected a,b] [-guard a,b] <input>\n eval -root <dir> [-protected a,b] [-guard a,b] [-metrics-out <file|->] <corpus.json>\n catalog -root <dir> [-protected a,b] [-guard a,b] 输出能力目录(注入规划者/领导提示词用)")
}
type commonOpts struct {
@@ -243,16 +244,20 @@ func sortedKeys(m map[string]string) []string {
return keys
}
// evalCorpus 的 record_only 声明语料为基线评测集:期望描述理想行为,
// 差异照常打印并计入指标,但不使 eval 失败(不设质量阈值)。
type evalCorpus struct {
Cases []intent.CorpusCase `json:"cases"`
BuiltinSpecs []intent.IntentSpec `json:"builtin_specs,omitempty"`
GuardDomains []string `json:"guard_domains,omitempty"`
ProtectedDomains []string `json:"protected_domains,omitempty"`
Cases []intent.CorpusCase `json:"cases"`
BuiltinSpecs []intent.IntentSpec `json:"builtin_specs,omitempty"`
GuardDomains []string `json:"guard_domains,omitempty"`
ProtectedDomains []string `json:"protected_domains,omitempty"`
RecordOnly bool `json:"record_only,omitempty"`
}
func cmdEval(args []string) error {
fs := flag.NewFlagSet("eval", flag.ContinueOnError)
c := parseCommon(fs)
metricsOut := fs.String("metrics-out", "", "指标 JSON 输出路径(- 为标准输出;留空不输出)")
if err := fs.Parse(args); err != nil {
return err
}
@@ -356,8 +361,36 @@ func cmdEval(args []string) error {
rs = fmt.Sprintf("%.3f", r)
}
fmt.Printf("指标: precision=%s recall=%s(TP=%d FP=%d FN=%d)\n", ps, rs, metrics.TP, metrics.FP, metrics.FN)
summary := intent.SummarizeCorpus(intent.CorpusDigest(data), corpus.Cases, results)
fmt.Print(summary.Table())
if *metricsOut != "" {
if err := writeMetrics(*metricsOut, summary); err != nil {
return err
}
}
if failed > 0 {
if corpus.RecordOnly {
fmt.Printf("基线语料(record_only):%d/%d 用例与理想期望不符,只记录不判失败\n", failed, len(corpus.Cases))
return nil
}
return fmt.Errorf("%d/%d 用例失败", failed, len(corpus.Cases))
}
return nil
}
func writeMetrics(path string, summary intent.EvalSummary) error {
out, err := json.MarshalIndent(summary, "", " ")
if err != nil {
return err
}
out = append(out, '\n')
if path == "-" {
_, err = os.Stdout.Write(out)
return err
}
if err := os.WriteFile(path, out, 0o644); err != nil {
return fmt.Errorf("写指标文件失败:%w", err)
}
fmt.Printf("指标 JSON 已写入 %s\n", path)
return nil
}
+39
View File
@@ -1,9 +1,13 @@
package main
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"agent-intent/intent"
)
const testSpecJSON = `{
@@ -71,3 +75,38 @@ func TestEvalFailsOnWrongExpectation(t *testing.T) {
t.Fatal("expected eval failure on wrong expectation")
}
}
func TestEvalRecordOnlyWritesMetrics(t *testing.T) {
dir := writeTemp(t, "specs.json", testSpecJSON)
tmp := t.TempDir()
corpus := filepath.Join(tmp, "corpus.json")
doc := `{"record_only": true, "cases": [
{"name": "miss", "category": "negation_reference", "input": "不用查进度了", "expect_branch": "fallback"},
{"name": "hit", "input": "天气不错", "expect_branch": "fallback"}]}`
if err := os.WriteFile(corpus, []byte(doc), 0o644); err != nil {
t.Fatal(err)
}
out := filepath.Join(tmp, "metrics.json")
if err := run([]string{"eval", "-root", dir, "-protected", "", "-metrics-out", out, corpus}); err != nil {
t.Fatalf("record_only eval should not fail: %v", err)
}
data, err := os.ReadFile(out)
if err != nil {
t.Fatal(err)
}
var s intent.EvalSummary
if err := json.Unmarshal(data, &s); err != nil {
t.Fatal(err)
}
if s.Schema != intent.EvalMetricsSchema || s.Overall.Cases != 2 || len(s.Categories) != 2 || s.Overall.MisExecutions != 1 {
t.Fatalf("summary = %+v", s)
}
strict := filepath.Join(tmp, "strict.json")
if err := os.WriteFile(strict, []byte(strings.Replace(doc, `"record_only": true, `, "", 1)), 0o644); err != nil {
t.Fatal(err)
}
if err := run([]string{"eval", "-root", dir, "-protected", "", strict}); err == nil {
t.Fatal("corpus without record_only must still fail on mismatch")
}
}
+85
View File
@@ -0,0 +1,85 @@
# 能力召回评测基线(A3)
本文记录召回评测集在当前识别器上的**指标基线**。只记录,不设质量阈值、不做通过判定;
据此决定后续方向(检索、模型调优等)不在本基线范围内。机器可读基线为同目录
[`metrics.json`](metrics.json)(schema `agent-intent.eval-metrics/v1`),本表数字与之一致。
## 语料
- 位置:[`intent/testdata/recall_corpus.json`](../intent/testdata/recall_corpus.json),
与 `intent/testdata/corpus.json`、`corpus_multiturn.json` 同一格式(`CorpusCase`),
由 `intentctl eval` 与 `go test` 共用。
- 摘要:`sha256:53d26cfc4caad994074db7d49924e3eabcba7c8ada4d6389a9a23f3085bc2885`
(语料文件原始字节的 sha256;语料变更后须重跑并更新本文与 `metrics.json`)。
- 数据:全部为合成用例,只复用测试夹具 `intent/testdata/specs/base.json` 的域、动作与槽位形态,
以及语料自声明的守卫 spec `guard.config_change`;不含真实用户数据。
- 期望写的是**理想行为**(不是当前行为的快照),语料声明 `record_only: true`:
与期望不符的用例照常打印 FAIL 并计入指标,但不使 eval 失败。
- 离线:生成路径用确定性回放生成器(回放用例 `generator_commands`),无模型、无网络调用。
### 类别覆盖
| 类别 | 用例 | 轮次 | 覆盖内容 |
|---|---|---|---|
| `negation_reference` | 8 | 10 | 否定指令(不要/别/不用/先不要)、`excludes_any` 否定、「还是刚才那个」类指代(单轮无上下文与多轮) |
| `multi_intent` | 7 | 7 | 一句含两个意图信号:动作+动作(应澄清)、寒暄+动作(应取动作)、动作+守卫(应守卫拦截) |
| `wrong_slot` | 7 | 7 | 槽位缺失、中文数字、两个候选值、越出字符集、仅符号、空主题 |
| `expired_confirmation` | 7 | 14 | 上一轮挂起预期(链式或宿主声明)在下一轮到达前过期:裸回答应回落、完整新指令应走规则 matched 而非 expectation_hit;含 1 例未过期对照 |
| `generation_consistency` | 8 | 8 | 同一输入同时走规则路径与生成路径(回放生成器 + 识别器兜底同款校验),比对 Top 与槽位;含槽位分歧、错动作、幻觉动作、闲聊 |
| 合计 | 37 | 46 | |
## 指标口径
按轮次计(v1 单轮用例即一轮);比率保留 4 位,分母为 0 记 `-`(JSON `null`)。
- **pass**:全部轮次与理想期望一致的用例数。
- **top1**:写了 `expect_top` 的轮次中,实际 Top 等于期望的比例。
- **mis_exec(误执行)**:实际 `matched`/`expectation_hit` 出了 Top,但与期望动作不同
(期望别的 Top,或期望 `fallback`/`ambiguous`/`guard_hit` 却执行了)——危险项。
- **clarify(澄清率)**:`guard_hit` 或产出 `clarify` 命令的轮次占比。
- **fallback(回落率)**:实际 `fallback` 的轮次占比。
- **gen_agree(生成一致率)**:`generation_consistency` 用例中规则路径与生成路径
Top ID 与槽位都一致的比例(两边都无 Top 也算一致)。
## 基线(本次运行)
| 类别 | 用例 | pass | 轮次 | top1 | mis_exec | clarify | fallback | gen_agree |
|---|---|---|---|---|---|---|---|---|
| `expired_confirmation` | 7 | 7 | 14 | 11/11 = 1.0000 | 0 | 5/14 = 0.3571 | 3/14 = 0.2143 | - |
| `generation_consistency` | 8 | 8 | 8 | 7/7 = 1.0000 | 0 | 0/8 = 0.0000 | 1/8 = 0.1250 | 5/8 = 0.6250 |
| `multi_intent` | 7 | 6 | 7 | 2/3 = 0.6667 | 1 | 5/7 = 0.7143 | 0/7 = 0.0000 | - |
| `negation_reference` | 8 | 3 | 10 | 3/3 = 1.0000 | 5 | 0/10 = 0.0000 | 2/10 = 0.2000 | - |
| `wrong_slot` | 7 | 3 | 7 | 7/7 = 1.0000 | 0 | 3/7 = 0.4286 | 0/7 = 0.0000 | - |
| **overall** | 37 | 27 | 46 | 30/31 = 0.9677 | 6 | 13/46 = 0.2826 | 6/46 = 0.1304 | 5/8 = 0.6250 |
同次运行的既有 I2 指标:precision=0.967 recall=0.967(TP=29 FP=1 FN=1)。
### 记录到的差异(事实,不含判定)
- `negation_reference`:5 次误执行均为语义否定被当作肯定——「不要重启任务」「别取消任务 42」
「不用查进度了」「先不要发布」「刚才那个任务先别重启」分别 matched 到 restart/cancel/status_query/deploy/restart;
只有 `excludes_any` 声明的否定词(「配置」)被拦下。
- `multi_intent`:「你好,帮我生成报告,主题是电机」按 priority 平分裁决到 `demo.greet.hello`(1 次误执行);
两个动作并列的输入均进入 ambiguous 澄清。
- `wrong_slot`:两个候选值(「任务 42 和任务 43」「任务 42 还是任务 43」)、越出字符集(`abc_def` 抽成 `abc`)、
仅符号(`--`)都直接 start、未澄清;缺失与中文数字正确进入 clarify。
- `expired_confirmation`:过期后裸回答回落、完整指令走规则 matched,均与期望一致。
- `generation_consistency`:3 例不一致——生成槽位 `run_id=43` 与规则 `42` 不同、生成错动作
`taskrun.restart`(规则为 status_query)、幻觉动作 `ghost.flow` 被校验 fail-closed 后无 Top。
## 重跑
在仓库根(依赖子模块已按 gitlink 初始化):
```bash
# 打印逐例 PASS/FAIL、指标表,并写机器基线(-metrics-out - 则输出到标准输出)
GOWORK=off go run ./cmd/intentctl eval -root intent/testdata/specs \
-metrics-out evaluation/metrics.json intent/testdata/recall_corpus.json
# 结构覆盖测试(≥30 例、五类齐全、每例可执行、指标 JSON 合法且与计数一致、基线摘要与语料一致)
GOWORK=off go test -count=1 -run 'TestRecallCorpusStructure' -v ./intent/
```
`-root` 指向测试同款夹具目录;守卫 spec 与守卫/保护域由语料 `builtin_specs`、`guard_domains`、
`protected_domains` 自声明,测试按同样方式建快照,因此两条命令的指标同源。
识别是确定性的:同语料、同代码重跑得到逐字节相同的 `metrics.json`。
+108
View File
@@ -0,0 +1,108 @@
{
"schema": "agent-intent.eval-metrics/v1",
"corpus_digest": "sha256:53d26cfc4caad994074db7d49924e3eabcba7c8ada4d6389a9a23f3085bc2885",
"overall": {
"category": "overall",
"cases": 37,
"passed_cases": 27,
"turns": 46,
"top1_expected": 31,
"top1_hits": 30,
"top1_hit_rate": 0.9677,
"mis_executions": 6,
"clarifications": 13,
"clarification_rate": 0.2826,
"fallbacks": 6,
"fallback_rate": 0.1304,
"generation_cases": 8,
"generation_agree": 5,
"generation_agreement_rate": 0.625
},
"categories": [
{
"category": "expired_confirmation",
"cases": 7,
"passed_cases": 7,
"turns": 14,
"top1_expected": 11,
"top1_hits": 11,
"top1_hit_rate": 1,
"mis_executions": 0,
"clarifications": 5,
"clarification_rate": 0.3571,
"fallbacks": 3,
"fallback_rate": 0.2143,
"generation_cases": 0,
"generation_agree": 0,
"generation_agreement_rate": null
},
{
"category": "generation_consistency",
"cases": 8,
"passed_cases": 8,
"turns": 8,
"top1_expected": 7,
"top1_hits": 7,
"top1_hit_rate": 1,
"mis_executions": 0,
"clarifications": 0,
"clarification_rate": 0,
"fallbacks": 1,
"fallback_rate": 0.125,
"generation_cases": 8,
"generation_agree": 5,
"generation_agreement_rate": 0.625
},
{
"category": "multi_intent",
"cases": 7,
"passed_cases": 6,
"turns": 7,
"top1_expected": 3,
"top1_hits": 2,
"top1_hit_rate": 0.6667,
"mis_executions": 1,
"clarifications": 5,
"clarification_rate": 0.7143,
"fallbacks": 0,
"fallback_rate": 0,
"generation_cases": 0,
"generation_agree": 0,
"generation_agreement_rate": null
},
{
"category": "negation_reference",
"cases": 8,
"passed_cases": 3,
"turns": 10,
"top1_expected": 3,
"top1_hits": 3,
"top1_hit_rate": 1,
"mis_executions": 5,
"clarifications": 0,
"clarification_rate": 0,
"fallbacks": 2,
"fallback_rate": 0.2,
"generation_cases": 0,
"generation_agree": 0,
"generation_agreement_rate": null
},
{
"category": "wrong_slot",
"cases": 7,
"passed_cases": 3,
"turns": 7,
"top1_expected": 7,
"top1_hits": 7,
"top1_hit_rate": 1,
"mis_executions": 0,
"clarifications": 3,
"clarification_rate": 0.4286,
"fallbacks": 0,
"fallback_rate": 0,
"generation_cases": 0,
"generation_agree": 0,
"generation_agreement_rate": null
}
]
}
+4 -3
View File
@@ -33,6 +33,7 @@ type CatalogSpec struct {
// Catalog 返回能力目录:全部启用的非守卫 spec,按合并序(确定性)。
// 守卫域 spec 是拦截器而非可调用能力,不进目录;需要全量时用 SpecIDs/SpecByID。
// 返回值为深拷贝:调用方改切片/枚举/metadata 不会改到快照。
func (s *Snapshot) Catalog() []CatalogSpec {
out := make([]CatalogSpec, 0, len(s.specs))
for _, cs := range s.specs {
@@ -55,7 +56,7 @@ func (s *Snapshot) Catalog() []CatalogSpec {
for _, sl := range cs.slots {
view.Slots = append(view.Slots, CatalogSlot{
Key: sl.key, Required: sl.required, Label: sl.label, Question: sl.question,
Enum: sl.enum, Format: specSlotFormat(cs.spec, sl.key),
Enum: cloneStrings(sl.enum), Format: specSlotFormat(cs.spec, sl.key),
})
}
out = append(out, view)
@@ -124,7 +125,7 @@ func specSlotFormat(spec IntentSpec, key string) string {
// CatalogTopK 取目录的前 k 条:按 Priority 降序、ID 升序(确定性排序;
// 排序口径是 priority,不是语义相关度——相关度排序由宿主经 examples 自行计算)。
// k <= 0 时等同 Catalog() 全量。
// k <= 0 时等同 Catalog() 全量。返回值与 Catalog 一样是深拷贝,不共享底层数组。
func (s *Snapshot) CatalogTopK(k int) []CatalogSpec {
all := s.Catalog()
if k <= 0 || len(all) <= k {
@@ -141,5 +142,5 @@ func (s *Snapshot) CatalogTopK(k int) []CatalogSpec {
}
return all[i].ID < all[j].ID
})
return all[:k]
return cloneCatalogSpecs(all[:k])
}
+116 -13
View File
@@ -7,22 +7,33 @@ import (
// CorpusCase 是评测用例:v1 单轮(input/expect_*)与 v2 多轮(turns)共用。
// v1 字段在 Turns 为空时生效,并自动等价于一个单 turn 用例(不含命令断言)。
// Category 与 GeneratorCommands 为可选字段(A3 召回评测):旧语料不写即按
// 未分类处理;GeneratorCommands 是确定性回放生成器的脚本输出,写了才在
// 首轮输入上额外跑一遍生成路径并与规则路径比对。
type CorpusCase struct {
Name string `json:"name"`
Category string `json:"category,omitempty"`
Input string `json:"input,omitempty"`
ExpectBranch string `json:"expect_branch,omitempty"`
ExpectTop string `json:"expect_top,omitempty"`
ExpectSlots map[string]string `json:"expect_slots,omitempty"`
ExpectMissingSlots []string `json:"expect_missing_slots,omitempty"`
Turns []CorpusTurn `json:"turns,omitempty"`
GeneratorCommands []Command `json:"generator_commands,omitempty"`
}
// CorpusTurn 是多轮用例中的一轮:输入与逐轮期望(分支/top/命令列表)。
// CorpusTurn 是多轮用例中的一轮:输入与逐轮期望(分支/top/槽位/命令列表)。
// PendingExpectation 声明本轮到达前宿主挂起的预期意图(覆盖上一轮的链式预期);
// ExpectationExpired 表示本轮到达时挂起预期已过期(声明的或链式的都作废)。
type CorpusTurn struct {
Input string `json:"input"`
ExpectBranch string `json:"expect_branch,omitempty"`
ExpectTop string `json:"expect_top,omitempty"`
ExpectCommands []CommandExpect `json:"expect_commands,omitempty"`
Input string `json:"input"`
PendingExpectation []string `json:"pending_expectation,omitempty"`
ExpectationExpired bool `json:"expectation_expired,omitempty"`
ExpectBranch string `json:"expect_branch,omitempty"`
ExpectTop string `json:"expect_top,omitempty"`
ExpectSlots map[string]string `json:"expect_slots,omitempty"`
ExpectMissingSlots []string `json:"expect_missing_slots,omitempty"`
ExpectCommands []CommandExpect `json:"expect_commands,omitempty"`
}
// CommandExpect 是对一条命令的期望;只有写出的字段参与断言。
@@ -44,14 +55,29 @@ type TurnResult struct {
Input string
Branch string
TopID string
Slots map[string]string
Commands []Command
Mismatches []string
}
// GeneratorComparison 是同一输入的规则路径与生成路径(确定性回放生成器,
// 经与识别器兜底同款的校验)的 Top 比对。Agree = Top ID 与槽位都一致
// (两边都没有 Top 也算一致)。
type GeneratorComparison struct {
Input string
RulesTopID string
GeneratorTopID string
RulesSlots map[string]string
GeneratorSlots map[string]string
Agree bool
}
// CaseResult 一个用例的全部轮次结果。Passed = 所有轮次无差异。
// Generator 仅在用例写了 generator_commands 时非 nil;比对结果不计入 Passed。
type CaseResult struct {
Name string
Turns []TurnResult
Name string
Turns []TurnResult
Generator *GeneratorComparison
}
// Passed 报告用例是否全部轮次通过。
@@ -74,17 +100,31 @@ func (c *chainedExpectation) Pending(ctx context.Context) *PendingExpectation {
return c.next
}
// RunCorpusCase 执行一个用例的全部轮次(v1 用例自动包装为单轮)。
// 识别器按轮次构建在链式预期上,保证 SetSlot+Start 这类组合可在评测中到达。
func RunCorpusCase(ctx context.Context, snap *Snapshot, tc CorpusCase) CaseResult {
turns := tc.Turns
if len(turns) == 0 {
turns = []CorpusTurn{{Input: tc.Input, ExpectBranch: tc.ExpectBranch, ExpectTop: tc.ExpectTop}}
// corpusTurns 返回用例的轮次(v1 用例自动包装为单轮,携带 v1 槽位期望)。
func corpusTurns(tc CorpusCase) []CorpusTurn {
if len(tc.Turns) > 0 {
return tc.Turns
}
return []CorpusTurn{{Input: tc.Input, ExpectBranch: tc.ExpectBranch, ExpectTop: tc.ExpectTop,
ExpectSlots: tc.ExpectSlots, ExpectMissingSlots: tc.ExpectMissingSlots}}
}
// RunCorpusCase 执行一个用例的全部轮次(v1 用例自动包装为单轮)。
// 识别器按轮次构建在链式预期上,保证 SetSlot+Start 这类组合可在评测中到达;
// 轮次可显式声明挂起预期或声明其已过期。写了 generator_commands 的用例
// 另在首轮输入上跑生成路径并与规则路径比对(离线、确定性回放)。
func RunCorpusCase(ctx context.Context, snap *Snapshot, tc CorpusCase) CaseResult {
turns := corpusTurns(tc)
exp := &chainedExpectation{}
rec := NewRecognizer(Options{}, exp, nil)
out := CaseResult{Name: tc.Name}
for i, turn := range turns {
if len(turn.PendingExpectation) > 0 {
exp.next = &PendingExpectation{IntentIDs: append([]string(nil), turn.PendingExpectation...), Reason: "eval declared"}
}
if turn.ExpectationExpired {
exp.next = nil
}
tr := TurnResult{Index: i, Input: turn.Input}
res, err := rec.Recognize(ctx, turn.Input, snap)
if err != nil {
@@ -95,6 +135,7 @@ func RunCorpusCase(ctx context.Context, snap *Snapshot, tc CorpusCase) CaseResul
tr.Branch = string(res.Branch)
if res.Top != nil {
tr.TopID = res.Top.ID
tr.Slots = res.Top.Slots
}
tr.Commands = res.Commands
tr.Mismatches = matchTurn(turn, res)
@@ -104,9 +145,52 @@ func RunCorpusCase(ctx context.Context, snap *Snapshot, tc CorpusCase) CaseResul
exp.next = &PendingExpectation{IntentIDs: []string{res.Top.ID}, Reason: "eval chained"}
}
}
if len(tc.GeneratorCommands) > 0 && len(out.Turns) > 0 {
out.Generator = compareGeneratorPath(ctx, snap, out.Turns[0], tc.GeneratorCommands)
}
return out
}
// replayGenerator 是评测用确定性生成器:忽略输入与目录,回放用例脚本中的命令
// (无模型、无网络)。
type replayGenerator struct {
cmds []Command
}
func (g replayGenerator) Generate(ctx context.Context, input string, catalog []CatalogSpec) ([]Command, error) {
out := make([]Command, len(g.cmds))
for i, c := range g.cmds {
c.Slots = cloneStringMap(c.Slots)
out[i] = c
}
return out, nil
}
// compareGeneratorPath 绕过规则阶段,直接走识别器兜底同款的生成路径
// (目录 top-k → 生成 → 校验 → 解析回权威 spec),再与规则路径首轮比对。
func compareGeneratorPath(ctx context.Context, snap *Snapshot, rules TurnResult, cmds []Command) *GeneratorComparison {
cmp := &GeneratorComparison{Input: rules.Input, RulesTopID: rules.TopID, RulesSlots: rules.Slots}
rec := NewRecognizer(Options{Generator: replayGenerator{cmds: cmds}}, nil, nil)
if res, ok := rec.generateCommands(ctx, rules.Input, snap); ok && res.Top != nil {
cmp.GeneratorTopID = res.Top.ID
cmp.GeneratorSlots = res.Top.Slots
}
cmp.Agree = cmp.RulesTopID == cmp.GeneratorTopID && sameSlots(cmp.RulesSlots, cmp.GeneratorSlots)
return cmp
}
func sameSlots(a, b map[string]string) bool {
if len(a) != len(b) {
return false
}
for k, v := range a {
if w, ok := b[k]; !ok || w != v {
return false
}
}
return true
}
func matchTurn(turn CorpusTurn, res Result) []string {
var mm []string
if turn.ExpectBranch != "" && string(res.Branch) != turn.ExpectBranch {
@@ -121,6 +205,24 @@ func matchTurn(turn CorpusTurn, res Result) []string {
mm = append(mm, fmt.Sprintf("top=%s(期望 %s)", got, turn.ExpectTop))
}
}
for _, k := range sortedKeys(turn.ExpectSlots) {
got := ""
if res.Top != nil {
got = res.Top.Slots[k]
}
if got != turn.ExpectSlots[k] {
mm = append(mm, fmt.Sprintf("slots[%s]=%q(期望 %q)", k, got, turn.ExpectSlots[k]))
}
}
if len(turn.ExpectMissingSlots) > 0 {
var got []string
if res.Top != nil {
got = res.Top.MissingRequiredSlots
}
if fmt.Sprint(got) != fmt.Sprint(turn.ExpectMissingSlots) {
mm = append(mm, fmt.Sprintf("missing=%v(期望 %v)", got, turn.ExpectMissingSlots))
}
}
if turn.ExpectCommands != nil {
mm = append(mm, matchCommands(turn.ExpectCommands, res.Commands)...)
}
@@ -208,6 +310,7 @@ func (m CorpusMetrics) Recall() (float64, bool) {
// - FP:实际 matched/top=Y 但期望的标签不是 matched/top=Y
// (期望别的 top、期望 fallback/ambiguous、或期望 matched 但 top 不同);
// - FN:期望 matched/top=X 但实际没拿到它(branch 不对或 top 不同)。
//
// 只有写了 expect_top 的轮次进入指标(意图识别通行口径)。
func MetricsOf(cases []CorpusCase, results []CaseResult) CorpusMetrics {
var m CorpusMetrics
+12
View File
@@ -75,4 +75,16 @@ func TestEvalCorpus(t *testing.T) {
_, snap := buildDemoRegistry(t)
rec := NewRecognizer(Options{}, nil, nil)
runCorpus(t, rec, snap, doc.Cases)
var shared struct {
Cases []CorpusCase `json:"cases"`
}
if err := json.Unmarshal(data, &shared); err != nil {
t.Fatal(err)
}
var results []CaseResult
for _, tc := range shared.Cases {
results = append(results, RunCorpusCase(context.Background(), snap, tc))
}
t.Log("\n" + SummarizeCorpus(CorpusDigest(data), shared.Cases, results).Table())
}
+193
View File
@@ -0,0 +1,193 @@
package intent
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"math"
"sort"
"strings"
)
// EvalMetricsSchema 是机器可读评测指标(EvalSummary JSON)的格式标识。
const EvalMetricsSchema = "agent-intent.eval-metrics/v1"
// UncategorizedCategory 是未写 category 的用例归入的类别(旧语料兼容)。
const UncategorizedCategory = "uncategorized"
// CategoryMetrics 是一个类别(或全量)的评测指标(A3 召回基线)。
// 计数口径按轮次(v1 用例即一轮),比率的分母为 0 时为 nil(JSON null):
// - top1:写了 expect_top 的轮次中实际 Top 等于期望的比例;
// - mis_executions:实际 matched/expectation_hit 出 Top,但与期望动作不同
// (期望别的 top,或期望 fallback/ambiguous/guard_hit 却执行了)——危险项;
// - clarification:guard_hit 或产出 clarify 命令的轮次;
// - fallback:实际 fallback 的轮次;
// - generation:写了 generator_commands 的用例中,规则路径与生成路径一致的比例。
type CategoryMetrics struct {
Category string `json:"category"`
Cases int `json:"cases"`
PassedCases int `json:"passed_cases"`
Turns int `json:"turns"`
Top1Expected int `json:"top1_expected"`
Top1Hits int `json:"top1_hits"`
Top1HitRate *float64 `json:"top1_hit_rate"`
MisExecutions int `json:"mis_executions"`
Clarifications int `json:"clarifications"`
ClarificationRate *float64 `json:"clarification_rate"`
Fallbacks int `json:"fallbacks"`
FallbackRate *float64 `json:"fallback_rate"`
GenerationCases int `json:"generation_cases"`
GenerationAgree int `json:"generation_agree"`
GenerationAgreementRate *float64 `json:"generation_agreement_rate"`
}
// EvalSummary 是一次评测的指标汇总:按类别(名称升序)与全量,附语料摘要。
// 只记录基线,不含阈值与通过判定。
type EvalSummary struct {
Schema string `json:"schema"`
CorpusDigest string `json:"corpus_digest"`
Overall CategoryMetrics `json:"overall"`
Categories []CategoryMetrics `json:"categories"`
}
// CorpusDigest 返回语料文件原始字节的摘要(sha256:<hex>)。
func CorpusDigest(data []byte) string {
sum := sha256.Sum256(data)
return "sha256:" + hex.EncodeToString(sum[:])
}
// SummarizeCorpus 汇总评测指标。results 须与 cases 按下标一一对应
// (即 RunCorpusCase 逐个执行 cases 的结果)。
func SummarizeCorpus(digest string, cases []CorpusCase, results []CaseResult) EvalSummary {
byCat := map[string]*CategoryMetrics{}
overall := &CategoryMetrics{Category: "overall"}
for i, tc := range cases {
if i >= len(results) {
break
}
cat := tc.Category
if strings.TrimSpace(cat) == "" {
cat = UncategorizedCategory
}
m, ok := byCat[cat]
if !ok {
m = &CategoryMetrics{Category: cat}
byCat[cat] = m
}
accumulateCase(m, tc, results[i])
accumulateCase(overall, tc, results[i])
}
out := EvalSummary{Schema: EvalMetricsSchema, CorpusDigest: digest, Categories: []CategoryMetrics{}}
for _, m := range byCat {
finishRates(m)
out.Categories = append(out.Categories, *m)
}
sort.Slice(out.Categories, func(i, j int) bool { return out.Categories[i].Category < out.Categories[j].Category })
finishRates(overall)
out.Overall = *overall
return out
}
func accumulateCase(m *CategoryMetrics, tc CorpusCase, res CaseResult) {
m.Cases++
if res.Passed() {
m.PassedCases++
}
turns := corpusTurns(tc)
for i, tr := range res.Turns {
if i >= len(turns) {
break
}
want := turns[i]
m.Turns++
if want.ExpectTop != "" {
m.Top1Expected++
if tr.TopID == want.ExpectTop {
m.Top1Hits++
}
}
if misExecuted(want, tr) {
m.MisExecutions++
}
if tr.Branch == string(BranchGuardHit) || hasClarify(tr.Commands) {
m.Clarifications++
}
if tr.Branch == string(BranchFallback) {
m.Fallbacks++
}
}
if res.Generator != nil {
m.GenerationCases++
if res.Generator.Agree {
m.GenerationAgree++
}
}
}
// misExecuted 报告该轮是否执行了与期望不同的动作(危险项)。
// 期望 matched/expectation_hit 但未写 expect_top 的轮次不约束动作,不计。
func misExecuted(want CorpusTurn, got TurnResult) bool {
executed := got.TopID != "" && (got.Branch == string(BranchMatched) || got.Branch == string(BranchExpectationHit))
if !executed {
return false
}
if want.ExpectTop != "" {
return got.TopID != want.ExpectTop
}
switch Branch(want.ExpectBranch) {
case BranchFallback, BranchAmbiguous, BranchGuardHit:
return true
}
return false
}
func hasClarify(cmds []Command) bool {
for _, c := range cmds {
if c.Kind == CommandClarify {
return true
}
}
return false
}
func finishRates(m *CategoryMetrics) {
m.Top1HitRate = ratio(m.Top1Hits, m.Top1Expected)
m.ClarificationRate = ratio(m.Clarifications, m.Turns)
m.FallbackRate = ratio(m.Fallbacks, m.Turns)
m.GenerationAgreementRate = ratio(m.GenerationAgree, m.GenerationCases)
}
// ratio 定点 4 位舍入,保证重跑 JSON 字节稳定;分母为 0 返回 nil。
func ratio(n, d int) *float64 {
if d == 0 {
return nil
}
v := math.Round(float64(n)/float64(d)*10000) / 10000
return &v
}
// Table 返回人类可读的指标表(每类别一行 + overall 行)。
func (s EvalSummary) Table() string {
var b strings.Builder
fmt.Fprintf(&b, "指标表(corpus %s):\n", s.CorpusDigest)
fmt.Fprintf(&b, " %-24s %5s %5s %5s %-14s %8s %-14s %-14s %s\n",
"category", "cases", "pass", "turns", "top1", "mis_exec", "clarify", "fallback", "gen_agree")
rows := append(append([]CategoryMetrics(nil), s.Categories...), s.Overall)
for _, m := range rows {
fmt.Fprintf(&b, " %-24s %5d %5d %5d %-14s %8d %-14s %-14s %s\n",
m.Category, m.Cases, m.PassedCases, m.Turns,
rateCell(m.Top1Hits, m.Top1Expected, m.Top1HitRate),
m.MisExecutions,
rateCell(m.Clarifications, m.Turns, m.ClarificationRate),
rateCell(m.Fallbacks, m.Turns, m.FallbackRate),
rateCell(m.GenerationAgree, m.GenerationCases, m.GenerationAgreementRate))
}
return b.String()
}
func rateCell(n, d int, r *float64) string {
if r == nil {
return "-"
}
return fmt.Sprintf("%d/%d=%.3f", n, d, *r)
}
+85 -4
View File
@@ -34,6 +34,26 @@ func cloneStrings(in []string) []string {
return append([]string(nil), in...)
}
func cloneStringMap(in map[string]string) map[string]string {
if in == nil {
return nil
}
out := make(map[string]string, len(in))
for k, v := range in {
out[k] = v
}
return out
}
func cloneActionRef(in *ActionRef) *ActionRef {
if in == nil {
return nil
}
cp := *in
return &cp
}
// cloneIntentSpec 深拷贝 IntentSpec,切断切片/map/指针与调用方或快照内部的共享。
func cloneIntentSpec(in IntentSpec) IntentSpec {
out := in
out.Matcher.IncludesAny = cloneStrings(in.Matcher.IncludesAny)
@@ -47,15 +67,76 @@ func cloneIntentSpec(in IntentSpec) IntentSpec {
out.Slots[i].Normalize = cloneStrings(in.Slots[i].Normalize)
}
}
if in.Metadata != nil {
out.Metadata = make(map[string]string, len(in.Metadata))
for k, v := range in.Metadata {
out.Metadata[k] = v
out.Metadata = cloneStringMap(in.Metadata)
out.Action = cloneActionRef(in.Action)
out.Examples = cloneStrings(in.Examples)
return out
}
func cloneCatalogSlot(in CatalogSlot) CatalogSlot {
out := in
out.Enum = cloneStrings(in.Enum)
return out
}
func cloneCatalogSpec(in CatalogSpec) CatalogSpec {
out := in
out.Metadata = cloneStringMap(in.Metadata)
if in.Slots != nil {
out.Slots = make([]CatalogSlot, len(in.Slots))
for i := range in.Slots {
out.Slots[i] = cloneCatalogSlot(in.Slots[i])
}
}
return out
}
func cloneCatalogSpecs(in []CatalogSpec) []CatalogSpec {
if in == nil {
return nil
}
out := make([]CatalogSpec, len(in))
for i := range in {
out[i] = cloneCatalogSpec(in[i])
}
return out
}
func cloneOptions(in Options) Options {
out := in
out.ProtectedDomains = cloneStrings(in.ProtectedDomains)
out.GuardDomains = cloneStrings(in.GuardDomains)
if in.SlotValidators != nil {
cp := make(map[string]SlotValidator, len(in.SlotValidators))
for k, v := range in.SlotValidators {
cp[k] = v
}
out.SlotValidators = cp
}
return out
}
func cloneCandidate(in Candidate) Candidate {
out := in
out.Metadata = cloneStringMap(in.Metadata)
out.Slots = cloneStringMap(in.Slots)
out.MissingRequiredSlots = cloneStrings(in.MissingRequiredSlots)
out.InvalidSlots = cloneStringMap(in.InvalidSlots)
out.Action = cloneActionRef(in.Action)
return out
}
func cloneCandidates(in []Candidate) []Candidate {
if in == nil {
return nil
}
out := make([]Candidate, len(in))
for i := range in {
out[i] = cloneCandidate(in[i])
}
return out
}
// catalogMetadataAllow 是目录与 Markdown 允许带出的 metadata 键。
// handler 仍留在 CatalogSpec 供执行器按名分派,但不写入 Markdown。
var catalogMetadataAllow = map[string]bool{
+4 -4
View File
@@ -7,10 +7,10 @@ import "context"
// 只在规则与语义都未命中(fallback)时被调用;实现自担延迟与成本。
// 返回错误时识别保持 fallback,不会使 Recognize 失败。
//
// 库侧校验与 deriveCommands 同款:start 命令的 ActionRef 必须可解析——
// auto/command/workflow/blueprint 的目标必须在 catalog 的 spec ID 集合内,
// handler 的目标必须在 catalog 的 Handler 集合内;槽位键必须在目标 spec 的
// 槽位键集合内(可解析时)。生成器幻觉一律 fail-closed 回 fallback。
// 库侧校验:start 的 ActionRef 必须能从同一快照唯一解析回权威 spec
// (spec ID、Action 别名、或唯一 handler 反查);槽位键必须在目标 spec
// 键集内,随后跑与规则路径同款的必填/枚举/format 校验并挂上元数据。
// 解析失败或未知键一律 fail-closed 回 fallback。
type CommandGenerator interface {
// Generate 基于输入与能力目录产出命令列表(catalog 由调用方按 top-k 给)。
Generate(ctx context.Context, input string, catalog []CatalogSpec) ([]Command, error)
+188
View File
@@ -0,0 +1,188 @@
package intent
import (
"bytes"
"context"
"encoding/json"
"os"
"strings"
"testing"
)
// recallCorpusDoc 与 intentctl eval 读取的语料结构一致(自声明守卫 spec 与域)。
type recallCorpusDoc struct {
RecordOnly bool `json:"record_only"`
Cases []CorpusCase `json:"cases"`
BuiltinSpecs []IntentSpec `json:"builtin_specs"`
GuardDomains []string `json:"guard_domains"`
ProtectedDomains []string `json:"protected_domains"`
}
var recallCategories = []string{
"expired_confirmation",
"generation_consistency",
"multi_intent",
"negation_reference",
"wrong_slot",
}
const (
recallCorpusPath = "testdata/recall_corpus.json"
recallBaselinePath = "../evaluation/metrics.json"
)
// runRecallCorpus 按 intentctl eval 同款方式建快照(语料 builtin_specs + 夹具 specs)
// 并逐例执行,保证测试与 CLI 的指标同源。
func runRecallCorpus(t *testing.T) ([]byte, recallCorpusDoc, []CaseResult) {
t.Helper()
data, err := os.ReadFile(recallCorpusPath)
if err != nil {
t.Fatal(err)
}
var doc recallCorpusDoc
if err := json.Unmarshal(data, &doc); err != nil {
t.Fatal(err)
}
b := NewRegistryBuilder(Options{GuardDomains: doc.GuardDomains, ProtectedDomains: doc.ProtectedDomains})
if err := b.Register(doc.BuiltinSpecs...); err != nil {
t.Fatal(err)
}
reg, err := b.BuildFS(context.Background(), os.DirFS("testdata"), "specs")
if err != nil {
t.Fatal(err)
}
snap := reg.Snapshot()
results := make([]CaseResult, 0, len(doc.Cases))
for _, tc := range doc.Cases {
results = append(results, RunCorpusCase(context.Background(), snap, tc))
}
return data, doc, results
}
// TestRecallCorpusStructure 只断言结构覆盖(A3):≥30 例、五类齐全、每例可执行、
// 指标 JSON 合法且与用例计数一致;不断言任何质量数值(基线只记录)。
func TestRecallCorpusStructure(t *testing.T) {
data, doc, results := runRecallCorpus(t)
if !doc.RecordOnly {
t.Fatal("召回基线语料须声明 record_only")
}
if len(doc.Cases) < 30 {
t.Fatalf("cases = %d, want >= 30", len(doc.Cases))
}
perCat := map[string]int{}
for i, tc := range doc.Cases {
perCat[tc.Category]++
turns := corpusTurns(tc)
res := results[i]
if len(res.Turns) != len(turns) {
t.Fatalf("case %s executed %d/%d turns", tc.Name, len(res.Turns), len(turns))
}
for _, tr := range res.Turns {
for _, mm := range tr.Mismatches {
if strings.HasPrefix(mm, "识别错误") {
t.Fatalf("case %s turn %d: %s", tc.Name, tr.Index, mm)
}
}
}
if tc.Category == "generation_consistency" {
if len(tc.GeneratorCommands) == 0 || len(turns) != 1 || res.Generator == nil {
t.Fatalf("case %s: generation_consistency 须为带 generator_commands 的单轮用例", tc.Name)
}
}
}
for _, cat := range recallCategories {
if perCat[cat] == 0 {
t.Fatalf("category %s missing (have %v)", cat, perCat)
}
}
if len(perCat) != len(recallCategories) {
t.Fatalf("unexpected categories %v, want exactly %v", perCat, recallCategories)
}
summary := SummarizeCorpus(CorpusDigest(data), doc.Cases, results)
t.Log("\n" + summary.Table())
raw, err := json.Marshal(summary)
if err != nil {
t.Fatal(err)
}
checkSummaryJSON(t, "fresh", raw, CorpusDigest(data), doc.Cases)
baseline, err := os.ReadFile(recallBaselinePath)
if err != nil {
t.Fatalf("read committed baseline: %v", err)
}
checkSummaryJSON(t, "baseline", baseline, CorpusDigest(data), doc.Cases)
}
// checkSummaryJSON 校验指标 JSON 结构与计数一致性(不比较质量数值)。
// 已提交基线的摘要必须等于当前语料摘要:改语料后须按 evaluation/BASELINE.md 重跑。
func checkSummaryJSON(t *testing.T, label string, raw []byte, digest string, cases []CorpusCase) {
t.Helper()
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
var s EvalSummary
if err := dec.Decode(&s); err != nil {
t.Fatalf("%s: metrics JSON malformed: %v", label, err)
}
if s.Schema != EvalMetricsSchema {
t.Fatalf("%s: schema = %q", label, s.Schema)
}
if s.CorpusDigest != digest {
t.Fatalf("%s: corpus_digest = %s, want %s(语料已变更,需重跑基线)", label, s.CorpusDigest, digest)
}
wantCases, wantTurns, wantGen := map[string]int{}, map[string]int{}, map[string]int{}
totalTurns, totalGen := 0, 0
for _, tc := range cases {
n := len(corpusTurns(tc))
wantCases[tc.Category]++
wantTurns[tc.Category] += n
totalTurns += n
if len(tc.GeneratorCommands) > 0 {
wantGen[tc.Category]++
totalGen++
}
}
if len(s.Categories) != len(wantCases) {
t.Fatalf("%s: categories = %d, want %d", label, len(s.Categories), len(wantCases))
}
sumCases, sumTurns := 0, 0
for _, m := range s.Categories {
if m.Cases != wantCases[m.Category] || m.Turns != wantTurns[m.Category] || m.GenerationCases != wantGen[m.Category] {
t.Fatalf("%s: %s counts = %+v, want cases=%d turns=%d gen=%d", label, m.Category, m, wantCases[m.Category], wantTurns[m.Category], wantGen[m.Category])
}
checkMetricsBounds(t, label, m)
sumCases += m.Cases
sumTurns += m.Turns
}
o := s.Overall
if o.Cases != len(cases) || sumCases != o.Cases || o.Turns != totalTurns || sumTurns != o.Turns || o.GenerationCases != totalGen {
t.Fatalf("%s: overall = %+v, want cases=%d turns=%d gen=%d", label, o, len(cases), totalTurns, totalGen)
}
checkMetricsBounds(t, label, o)
}
func checkMetricsBounds(t *testing.T, label string, m CategoryMetrics) {
t.Helper()
pairs := []struct {
name string
n, d int
rate *float64
}{
{"top1", m.Top1Hits, m.Top1Expected, m.Top1HitRate},
{"clarification", m.Clarifications, m.Turns, m.ClarificationRate},
{"fallback", m.Fallbacks, m.Turns, m.FallbackRate},
{"generation", m.GenerationAgree, m.GenerationCases, m.GenerationAgreementRate},
}
for _, p := range pairs {
if p.n < 0 || p.n > p.d {
t.Fatalf("%s: %s %s = %d/%d out of range", label, m.Category, p.name, p.n, p.d)
}
want := ratio(p.n, p.d)
if (want == nil) != (p.rate == nil) || (want != nil && *want != *p.rate) {
t.Fatalf("%s: %s %s rate inconsistent with %d/%d", label, m.Category, p.name, p.n, p.d)
}
}
if m.PassedCases > m.Cases || m.MisExecutions > m.Turns || m.Top1Expected > m.Turns {
t.Fatalf("%s: %s counts out of range: %+v", label, m.Category, m)
}
}
+137 -61
View File
@@ -120,7 +120,8 @@ func (r *FunnelRecognizer) recognize(ctx context.Context, input string, snap *Sn
}
}
if r.cls != nil {
refined, err := r.cls.Refine(ctx, input, cands)
// 传入候选副本:分类器就地改写不会污染原候选(失败时仍返回未篡改集)。
refined, err := r.cls.Refine(ctx, input, cloneCandidates(cands))
if err != nil {
r.log.Module("intent").WithContext(ctx).Warn("intent classifier failed; keep ambiguous", "error", err.Error())
return Result{Input: input, Branch: BranchAmbiguous, Candidates: cands, SnapshotRevision: snap.rev}
@@ -175,10 +176,11 @@ func validateRefined(original, refined []Candidate) ([]Candidate, error) {
if !ok {
return nil, fmt.Errorf("unknown refined id %s", got.ID)
}
if !validConfidence(got.Confidence) {
norm, ok := normalizeScore(got.Confidence)
if !ok {
return nil, fmt.Errorf("confidence out of range for %s", got.ID)
}
src.Confidence = got.Confidence
src.Confidence = norm
out[i] = src
}
return out, nil
@@ -188,6 +190,15 @@ func validConfidence(v float64) bool {
return !math.IsNaN(v) && !math.IsInf(v, 0) && v >= 0 && v <= 1
}
// normalizeScore 拒绝非有限/越界分数,并通过与规则计分同款的定点舍入后再交给阈值比较,
// 防止篡改或噪声分数绕过阈值进入 matched。
func normalizeScore(v float64) (float64, bool) {
if !validConfidence(v) {
return 0, false
}
return math.Round(v*10000) / 10000, true
}
// adjudicateRefined 对分类器结果重排后应用同一裁决规则;分差仍贴近则返回 false。
func adjudicateRefined(refined []Candidate, snap *Snapshot) (Candidate, bool) {
if len(refined) == 0 {
@@ -289,6 +300,7 @@ func sortRejected(rs []RejectedSpec) {
// 规则:候选必须来自当前候选集(构造上不可能引入未知 ID);
// 只取相似度 ≥ 阈值的候选——唯一过阈或前两名分差 ≥ margin 才判 matched,
// 否则保持 ambiguous(与规则裁决同款语义)。打分错误一律记日志并保持 ambiguous。
// examples 以副本传入打分器;非有限/越界分数拒绝;定点舍入后再与阈值比较。
func (r *FunnelRecognizer) semanticRerank(ctx context.Context, input string, cands []Candidate, snap *Snapshot) (Candidate, bool) {
type scored struct {
cand Candidate
@@ -300,14 +312,21 @@ func (r *FunnelRecognizer) semanticRerank(ctx context.Context, input string, can
if !ok || len(spec.Examples) == 0 {
continue
}
s, err := r.sem.Score(ctx, input, spec.Examples)
examples := cloneStrings(spec.Examples)
s, err := r.sem.Score(ctx, input, examples)
if err != nil {
r.log.Module("intent").WithContext(ctx).Warn("intent semantic scorer failed; keep ambiguous",
"error", err.Error(), "id", c.ID)
continue
}
if s >= r.thr {
above = append(above, scored{cand: c, score: s})
norm, ok := normalizeScore(s)
if !ok {
r.log.Module("intent").WithContext(ctx).Warn("intent semantic score rejected; keep ambiguous",
"id", c.ID, "score", s)
continue
}
if norm >= r.thr {
above = append(above, scored{cand: c, score: norm})
}
}
if len(above) == 0 {
@@ -327,9 +346,9 @@ func (r *FunnelRecognizer) semanticRerank(ctx context.Context, input string, can
}
// generateCommands 经模型命令生成端口兜底(I2)。校验:
// start 的 ActionRef 可解析(spec ID 集或 Handler 集内)、命令种类受限、
// 槽位键在目标 spec 的键集内(可解析时);全部通过则把首个 start 的目标
// spec 合成 Top(refined_by=generator 标注),任一项不过则保持 fallback。
// start 的 ActionRef 必须能从同一快照唯一解析回权威 spec(含 Action 别名与
// handler→spec 反查)、命令种类受限、槽位键合法;解析成功后对生成槽位跑与
// 规则路径同款的全量校验并挂上 risk/confirm/action 元数据。任一项不过则保持 fallback(无 Top)。
func (r *FunnelRecognizer) generateCommands(ctx context.Context, input string, snap *Snapshot) (Result, bool) {
catalog := snap.CatalogTopK(DefaultGeneratorCatalogK)
cmds, err := r.gen.Generate(ctx, input, catalog)
@@ -337,7 +356,7 @@ func (r *FunnelRecognizer) generateCommands(ctx context.Context, input string, s
r.log.Module("intent").WithContext(ctx).Warn("intent command generator failed; keep fallback", "error", err.Error())
return Result{}, false
}
if err := validateGenerated(cmds, catalog); err != nil {
if err := validateGenerated(cmds, snap); err != nil {
r.log.Module("intent").WithContext(ctx).Warn("intent generator output rejected; keep fallback", "error", err.Error())
return Result{}, false
}
@@ -346,16 +365,14 @@ func (r *FunnelRecognizer) generateCommands(ctx context.Context, input string, s
if c.Kind != CommandStart {
continue
}
if spec, ok := generatedSpecOf(c.Action, catalog); ok {
top := Candidate{
ID: spec.ID, Domain: spec.Domain, Version: spec.Version,
Confidence: 0, Reason: "generated_by=command_generator",
Slots: c.Slots,
}
res.Top = &top
res.Branch = BranchMatched
res.Candidates = []Candidate{top}
cs, err := snap.resolveActionRef(c.Action)
if err != nil || cs == nil {
break
}
top := candidateFromGenerated(cs, c.Slots)
res.Top = &top
res.Branch = BranchMatched
res.Candidates = []Candidate{top}
break
}
return res, true
@@ -364,46 +381,53 @@ func (r *FunnelRecognizer) generateCommands(ctx context.Context, input string, s
// DefaultGeneratorCatalogK 生成器目录默认 top-k。
const DefaultGeneratorCatalogK = 8
// candidateFromGenerated 用权威 spec 元数据 + 生成槽位的全量校验合成 Top(与规则路径同款字段)。
func candidateFromGenerated(cs *compiledSpec, provided map[string]string) Candidate {
c := Candidate{
ID: cs.spec.ID,
Domain: cs.spec.Domain,
Version: cs.spec.Version,
Confidence: 0,
Priority: cs.spec.Priority,
Risk: cs.spec.Risk,
RequiresConfirm: cs.spec.RequiresConfirm,
Handler: cs.spec.Handler,
Reason: "generated_by=command_generator",
}
if c.Risk == "" {
c.Risk = RiskLow
}
if cs.spec.Action != nil {
ref := *cs.spec.Action
c.Action = &ref
}
if len(cs.spec.Metadata) > 0 {
c.Metadata = cloneStringMap(cs.spec.Metadata)
}
c.Slots, c.MissingRequiredSlots, c.InvalidSlots = cs.validateProvidedSlots(provided)
return c
}
// validateGenerated 校验生成命令(I2;与 spec 级校验同款 fail-closed)。
func validateGenerated(cmds []Command, catalog []CatalogSpec) error {
// ActionRef 经快照唯一解析;槽位未知键在此拒绝(与规则路径「键集外」同款 fail-closed)。
func validateGenerated(cmds []Command, snap *Snapshot) error {
if len(cmds) == 0 {
return fmt.Errorf("empty command list")
}
specIDs := map[string]bool{}
handlers := map[string]bool{}
slotKeys := map[string]map[string]bool{}
for _, cs := range catalog {
specIDs[cs.ID] = true
if cs.Handler != "" {
handlers[cs.Handler] = true
}
keys := map[string]bool{}
for _, sl := range cs.Slots {
keys[sl.Key] = true
}
slotKeys[cs.ID] = keys
}
for i, c := range cmds {
switch c.Kind {
case CommandStart:
ref := c.Action
switch ref.Kind {
case ActionAuto, ActionCommand, ActionWorkflow, ActionBlueprint:
if !specIDs[ref.ID] {
return fmt.Errorf("cmd[%d] start: action.id %q 不在目录内", i, ref.ID)
}
case ActionHandler:
if !handlers[ref.ID] {
return fmt.Errorf("cmd[%d] start: handler %q 不在目录内", i, ref.ID)
}
default:
return fmt.Errorf("cmd[%d] start: 未知 action.kind %q", i, ref.Kind)
cs, err := snap.resolveActionRef(c.Action)
if err != nil {
return fmt.Errorf("cmd[%d] start: %w", i, err)
}
if keys, ok := slotKeys[ref.ID]; ok {
for k := range c.Slots {
if !keys[k] {
return fmt.Errorf("cmd[%d] start: 槽位 %q 不在 %s 的槽位键集内", i, k, ref.ID)
}
keys := map[string]bool{}
for _, sl := range cs.slots {
keys[sl.key] = true
}
for k := range c.Slots {
if !keys[k] {
return fmt.Errorf("cmd[%d] start: 槽位 %q 不在 %s 的槽位键集内", i, k, cs.spec.ID)
}
}
case CommandClarify, CommandCancel, CommandSetSlot, CommandChangeSlot, CommandChitchat, CommandHandoff:
@@ -414,18 +438,70 @@ func validateGenerated(cmds []Command, catalog []CatalogSpec) error {
return nil
}
// generatedSpecOf 把 start 的 ActionRef 解析回目录 spec(handler 键按 Handler 反查)。
func generatedSpecOf(ref ActionRef, catalog []CatalogSpec) (CatalogSpec, bool) {
for _, cs := range catalog {
if ref.Kind == ActionHandler {
if cs.Handler == ref.ID {
return cs, true
}
// resolveActionRef 把 start 的 ActionRef 解析回快照内唯一权威 compiledSpec。
// ActionCommand/Workflow/Blueprint/Auto:先按 spec ID,再按 spec.Action 声明的别名;
// ActionHandler:经 handler→spec 反查,重名拒绝。
func (s *Snapshot) resolveActionRef(ref ActionRef) (*compiledSpec, error) {
switch ref.Kind {
case ActionHandler:
return s.resolveByHandler(ref.ID)
case ActionAuto, ActionCommand, ActionWorkflow, ActionBlueprint:
return s.resolveBySpecOrAlias(ref)
default:
return nil, fmt.Errorf("未知 action.kind %q", ref.Kind)
}
}
func (s *Snapshot) resolveByHandler(handler string) (*compiledSpec, error) {
if strings.TrimSpace(handler) == "" {
return nil, fmt.Errorf("handler 为空")
}
var found *compiledSpec
for _, cs := range s.specs {
if cs.guard {
continue
}
if cs.ID == ref.ID {
return cs, true
if cs.spec.Handler != handler {
continue
}
if found != nil {
return nil, fmt.Errorf("handler %q 对应多个 spec(歧义)", handler)
}
found = cs
}
return CatalogSpec{}, false
if found == nil {
return nil, fmt.Errorf("handler %q 不在快照内", handler)
}
return found, nil
}
func (s *Snapshot) resolveBySpecOrAlias(ref ActionRef) (*compiledSpec, error) {
if strings.TrimSpace(ref.ID) == "" {
return nil, fmt.Errorf("action.id 为空")
}
var found *compiledSpec
for _, cs := range s.specs {
if cs.guard {
continue
}
match := cs.spec.ID == ref.ID
if !match && cs.spec.Action != nil && cs.spec.Action.Kind == ref.Kind && cs.spec.Action.ID == ref.ID {
match = true
}
// ActionAuto 也接受「合法别名」:显式 Action 的 ID 与 kind 匹配任一声明。
if !match && ref.Kind == ActionAuto && cs.spec.Action != nil && cs.spec.Action.ID == ref.ID {
match = true
}
if !match {
continue
}
if found != nil && found != cs {
return nil, fmt.Errorf("action.id %q 对应多个 spec(歧义)", ref.ID)
}
found = cs
}
if found == nil {
return nil, fmt.Errorf("action.id %q 不在快照内", ref.ID)
}
return found, nil
}
+17 -16
View File
@@ -60,7 +60,7 @@ const (
)
func (o Options) normalized() Options {
n := o
n := cloneOptions(o)
if n.TopN <= 0 {
n.TopN = DefaultTopN
}
@@ -85,13 +85,6 @@ func (o Options) normalized() Options {
if n.Log == nil {
n.Log = logsys.Nop()
}
if len(n.SlotValidators) > 0 {
cp := make(map[string]SlotValidator, len(n.SlotValidators))
for k, v := range n.SlotValidators {
cp[k] = v
}
n.SlotValidators = cp
}
return n
}
@@ -203,7 +196,7 @@ func compileSpec(spec IntentSpec, guard bool, index int, scoped map[string]SlotV
return out
}
cs := &compiledSpec{
spec: spec,
spec: cloneIntentSpec(spec),
termsAny: fold(m.IncludesAny),
termsAll: fold(m.IncludesAll),
termsEx: fold(m.ExcludesAny),
@@ -481,6 +474,7 @@ func NewRegistryBuilder(opts Options) *RegistryBuilder {
}
// Register 登记内置基线 spec,立即校验;重复 id 报 InvalidSpec。
// 登记时深拷贝:调用方之后改入参不会影响已登记定义。
func (b *RegistryBuilder) Register(specs ...IntentSpec) error {
lim := b.opts.limits()
for _, spec := range specs {
@@ -491,7 +485,7 @@ func (b *RegistryBuilder) Register(specs ...IntentSpec) error {
return err
}
b.seen[spec.ID] = true
b.builtin = append(b.builtin, spec)
b.builtin = append(b.builtin, cloneIntentSpec(spec))
}
return nil
}
@@ -526,9 +520,13 @@ func (b *RegistryBuilder) buildWithSource(fsys fs.FS, roots []string) (*Registry
if err != nil {
return nil, err
}
builtin := make([]IntentSpec, len(b.builtin))
for i, spec := range b.builtin {
builtin[i] = cloneIntentSpec(spec)
}
r := &Registry{
opts: b.opts,
builtin: append([]IntentSpec(nil), b.builtin...),
opts: cloneOptions(b.opts),
builtin: builtin,
fsys: fsys,
roots: append([]string(nil), roots...),
hasSource: fsys != nil,
@@ -553,8 +551,8 @@ type Registry struct {
// Snapshot 返回当前生效快照。永不为 nil。
func (r *Registry) Snapshot() *Snapshot { return r.snap.Load() }
// Options 返回生效选项(已填默认值)。
func (r *Registry) Options() Options { return r.opts }
// Options 返回生效选项(已填默认值)的深拷贝:改切片/校验器表不影响 Registry。
func (r *Registry) Options() Options { return cloneOptions(r.opts) }
// Reload 重读配置来源并构建新快照;成功后原子替换并返回新快照。
// 任一步失败保留旧快照,revision 不变,返回带来源与原因的错误。
@@ -590,7 +588,10 @@ func buildSnapshot(opts Options, baseline []IntentSpec, entries []overlayEntry,
guards := opts.guardSet()
lim := opts.limits()
ordered := append([]IntentSpec(nil), baseline...)
ordered := make([]IntentSpec, len(baseline))
for i, spec := range baseline {
ordered[i] = cloneIntentSpec(spec)
}
pos := make(map[string]int, len(ordered))
sources := make(map[string]string, len(ordered))
for i, spec := range ordered {
@@ -620,7 +621,7 @@ func buildSnapshot(opts Options, baseline []IntentSpec, entries []overlayEntry,
sources[e.id] = e.source + " (disabled)"
continue
}
spec := e.spec
spec := cloneIntentSpec(e.spec)
if err := validateSpec(spec, lim, opts.SlotValidators); err != nil {
return nil, err.(*Error).with("source", e.source)
}
+50
View File
@@ -167,6 +167,56 @@ func (cs *compiledSpec) extractSlots(input string) (slots map[string]string, mis
return slots, missing, invalid
}
// validateProvidedSlots 校验调用方直接给出的槽位表(生成器路径)。
// 与 extractSlots 同款结果形态:归一化 → format/enum/validator;必填缺失进 missing;
// 捕获到但不合法进 invalid;未知键进 invalid(reason=unknown slot key)。
// 非法或未知键的值不写入 slots。
func (cs *compiledSpec) validateProvidedSlots(provided map[string]string) (slots map[string]string, missing []string, invalid map[string]string) {
byKey := make(map[string]*compiledSlot, len(cs.slots))
for i := range cs.slots {
byKey[cs.slots[i].key] = &cs.slots[i]
}
for k, raw := range provided {
s, ok := byKey[k]
if !ok {
if invalid == nil {
invalid = map[string]string{}
}
invalid[k] = "unknown slot key"
continue
}
v := normalizeSlotValue(raw, s.normalize)
reason, valid := s.validate(v)
if !valid {
if invalid == nil {
invalid = map[string]string{}
}
invalid[k] = reason
continue
}
if slots == nil {
slots = map[string]string{}
}
slots[k] = v
}
for _, s := range cs.slots {
if !s.required {
continue
}
if _, ok := slots[s.key]; ok {
continue
}
// 必填且已标 invalid 的不重复进 missing(与 extractSlots 同款)。
if invalid != nil {
if _, bad := invalid[s.key]; bad {
continue
}
}
missing = append(missing, s.key)
}
return slots, missing, invalid
}
func compileSlot(spec SlotSpec, scoped map[string]SlotValidator) (compiledSlot, error) {
re, err := regexp.Compile(spec.Pattern)
if err != nil {
+575
View File
@@ -0,0 +1,575 @@
{
"record_only": true,
"cases": [
{
"name": "neg_do_not_restart",
"category": "negation_reference",
"input": "不要重启任务",
"expect_branch": "fallback"
},
{
"name": "neg_do_not_cancel_run",
"category": "negation_reference",
"input": "别取消任务 42",
"expect_branch": "fallback"
},
{
"name": "neg_no_need_progress",
"category": "negation_reference",
"input": "不用查进度了",
"expect_branch": "fallback"
},
{
"name": "neg_do_not_deploy",
"category": "negation_reference",
"input": "先不要发布",
"expect_branch": "fallback"
},
{
"name": "neg_excludes_config_progress",
"category": "negation_reference",
"input": "进度配置在哪里看",
"expect_branch": "fallback"
},
{
"name": "ref_same_as_before_no_context",
"category": "negation_reference",
"input": "还是刚才那个",
"expect_branch": "fallback"
},
{
"name": "ref_same_after_status",
"category": "negation_reference",
"turns": [
{
"input": "查询任务 42 的进度",
"expect_branch": "matched",
"expect_top": "demo.run.status_query",
"expect_slots": {
"run_id": "42"
}
},
{
"input": "还是刚才那个,再看下进度",
"expect_branch": "matched",
"expect_top": "demo.run.status_query"
}
]
},
{
"name": "ref_previous_task_hold_restart",
"category": "negation_reference",
"turns": [
{
"input": "查询任务 42 的进度",
"expect_branch": "matched",
"expect_top": "demo.run.status_query"
},
{
"input": "刚才那个任务先别重启",
"expect_branch": "fallback"
}
]
},
{
"name": "multi_status_then_restart",
"category": "multi_intent",
"input": "查询任务 42 的进度,然后重启任务",
"expect_branch": "ambiguous"
},
{
"name": "multi_greet_then_report",
"category": "multi_intent",
"input": "你好,帮我生成报告,主题是电机",
"expect_branch": "matched",
"expect_top": "report.generate",
"expect_slots": {
"topic": "电机"
}
},
{
"name": "multi_cancel_and_deploy",
"category": "multi_intent",
"input": "取消任务 42 并发布",
"expect_branch": "ambiguous"
},
{
"name": "multi_greet_then_status",
"category": "multi_intent",
"input": "你好,查一下任务 7 的进度",
"expect_branch": "matched",
"expect_top": "demo.run.status_query",
"expect_slots": {
"run_id": "7"
}
},
{
"name": "multi_restart_and_deploy",
"category": "multi_intent",
"input": "重启任务后再发布",
"expect_branch": "ambiguous"
},
{
"name": "multi_status_and_guard",
"category": "multi_intent",
"input": "查一下进度,顺便改密码",
"expect_branch": "guard_hit",
"expect_top": "guard.config_change"
},
{
"name": "multi_report_and_cancel",
"category": "multi_intent",
"input": "生成报告,主题是电机,再取消任务 9",
"expect_branch": "ambiguous"
},
{
"name": "slot_cancel_no_id",
"category": "wrong_slot",
"turns": [
{
"input": "取消任务",
"expect_branch": "matched",
"expect_top": "demo.run.cancel",
"expect_missing_slots": [
"run_id"
],
"expect_commands": [
{
"kind": "start"
},
{
"kind": "clarify",
"missing": [
"run_id"
]
}
]
}
]
},
{
"name": "slot_cancel_chinese_numeral",
"category": "wrong_slot",
"turns": [
{
"input": "取消任务四十二",
"expect_branch": "matched",
"expect_top": "demo.run.cancel",
"expect_missing_slots": [
"run_id"
],
"expect_commands": [
{
"kind": "start"
},
{
"kind": "clarify",
"missing": [
"run_id"
]
}
]
}
]
},
{
"name": "slot_cancel_two_ids",
"category": "wrong_slot",
"turns": [
{
"input": "取消任务 42 和任务 43",
"expect_branch": "matched",
"expect_top": "demo.run.cancel",
"expect_commands": [
{
"kind": "start"
},
{
"kind": "clarify"
}
]
}
]
},
{
"name": "slot_status_two_ids",
"category": "wrong_slot",
"turns": [
{
"input": "查询任务 42 还是任务 43 的进度",
"expect_branch": "matched",
"expect_top": "demo.run.status_query",
"expect_commands": [
{
"kind": "start"
},
{
"kind": "clarify"
}
]
}
]
},
{
"name": "slot_status_id_underscore",
"category": "wrong_slot",
"turns": [
{
"input": "查询任务 abc_def 的进度",
"expect_branch": "matched",
"expect_top": "demo.run.status_query",
"expect_commands": [
{
"kind": "start"
},
{
"kind": "clarify"
}
]
}
]
},
{
"name": "slot_cancel_dash_only",
"category": "wrong_slot",
"turns": [
{
"input": "取消任务 --",
"expect_branch": "matched",
"expect_top": "demo.run.cancel",
"expect_commands": [
{
"kind": "start"
},
{
"kind": "clarify"
}
]
}
]
},
{
"name": "slot_report_empty_topic",
"category": "wrong_slot",
"turns": [
{
"input": "生成报告,主题是",
"expect_branch": "matched",
"expect_top": "report.generate",
"expect_missing_slots": [
"topic"
],
"expect_commands": [
{
"kind": "start"
},
{
"kind": "clarify",
"missing": [
"topic"
],
"question": "要查询哪个主题?"
}
]
}
]
},
{
"name": "exp_cancel_live_control",
"category": "expired_confirmation",
"turns": [
{
"input": "取消一下",
"expect_branch": "matched",
"expect_top": "demo.run.cancel",
"expect_missing_slots": [
"run_id"
]
},
{
"input": "取消任务 42",
"expect_branch": "expectation_hit",
"expect_top": "demo.run.cancel",
"expect_slots": {
"run_id": "42"
}
}
]
},
{
"name": "exp_cancel_expired_full_command",
"category": "expired_confirmation",
"turns": [
{
"input": "取消一下",
"expect_branch": "matched",
"expect_top": "demo.run.cancel"
},
{
"input": "取消任务 42",
"expectation_expired": true,
"expect_branch": "matched",
"expect_top": "demo.run.cancel",
"expect_slots": {
"run_id": "42"
}
}
]
},
{
"name": "exp_cancel_expired_bare_answer",
"category": "expired_confirmation",
"turns": [
{
"input": "取消一下",
"expect_branch": "matched",
"expect_top": "demo.run.cancel"
},
{
"input": "就是任务 42",
"expectation_expired": true,
"expect_branch": "fallback"
}
]
},
{
"name": "exp_report_expired_bare_topic",
"category": "expired_confirmation",
"turns": [
{
"input": "帮我生成报告",
"expect_branch": "matched",
"expect_top": "report.generate"
},
{
"input": "主题是电机",
"expectation_expired": true,
"expect_branch": "fallback"
}
]
},
{
"name": "exp_report_expired_full_command",
"category": "expired_confirmation",
"turns": [
{
"input": "帮我生成报告",
"expect_branch": "matched",
"expect_top": "report.generate"
},
{
"input": "生成报告,主题是水泵",
"expectation_expired": true,
"expect_branch": "matched",
"expect_top": "report.generate",
"expect_slots": {
"topic": "水泵"
}
}
]
},
{
"name": "exp_restart_confirm_expired",
"category": "expired_confirmation",
"turns": [
{
"input": "把任务重启一下",
"expect_branch": "matched",
"expect_top": "demo.run.restart"
},
{
"input": "好的,确认",
"pending_expectation": [
"demo.run.restart"
],
"expectation_expired": true,
"expect_branch": "fallback"
}
]
},
{
"name": "exp_restart_confirm_expired_restated",
"category": "expired_confirmation",
"turns": [
{
"input": "把任务重启一下",
"expect_branch": "matched",
"expect_top": "demo.run.restart"
},
{
"input": "确认重启任务",
"pending_expectation": [
"demo.run.restart"
],
"expectation_expired": true,
"expect_branch": "matched",
"expect_top": "demo.run.restart"
}
]
},
{
"name": "gen_status_agree",
"category": "generation_consistency",
"input": "查询任务 42 的进度",
"expect_branch": "matched",
"expect_top": "demo.run.status_query",
"expect_slots": {
"run_id": "42"
},
"generator_commands": [
{
"kind": "start",
"action": {
"kind": "handler",
"id": "taskrun.query_status"
},
"slots": {
"run_id": "42"
}
}
]
},
{
"name": "gen_restart_agree",
"category": "generation_consistency",
"input": "把任务重启一下",
"expect_branch": "matched",
"expect_top": "demo.run.restart",
"generator_commands": [
{
"kind": "start",
"action": {
"kind": "auto",
"id": "demo.run.restart"
}
}
]
},
{
"name": "gen_report_agree",
"category": "generation_consistency",
"input": "生成报告,主题是电机",
"expect_branch": "matched",
"expect_top": "report.generate",
"expect_slots": {
"topic": "电机"
},
"generator_commands": [
{
"kind": "start",
"action": {
"kind": "handler",
"id": "reportHandler"
},
"slots": {
"topic": "电机"
}
}
]
},
{
"name": "gen_deploy_agree",
"category": "generation_consistency",
"input": "please DEPLOY it",
"expect_branch": "matched",
"expect_top": "demo.misc.deploy",
"generator_commands": [
{
"kind": "start",
"action": {
"kind": "handler",
"id": "deploy.run"
}
}
]
},
{
"name": "gen_cancel_slot_disagree",
"category": "generation_consistency",
"input": "取消任务 42",
"expect_branch": "matched",
"expect_top": "demo.run.cancel",
"expect_slots": {
"run_id": "42"
},
"generator_commands": [
{
"kind": "start",
"action": {
"kind": "auto",
"id": "demo.run.cancel"
},
"slots": {
"run_id": "43"
}
}
]
},
{
"name": "gen_progress_wrong_action",
"category": "generation_consistency",
"input": "帮我看下进度",
"expect_branch": "matched",
"expect_top": "demo.run.status_query",
"generator_commands": [
{
"kind": "start",
"action": {
"kind": "handler",
"id": "taskrun.restart"
}
}
]
},
{
"name": "gen_greet_hallucinated_action",
"category": "generation_consistency",
"input": "你好呀",
"expect_branch": "matched",
"expect_top": "demo.greet.hello",
"generator_commands": [
{
"kind": "start",
"action": {
"kind": "workflow",
"id": "ghost.flow"
}
}
]
},
{
"name": "gen_chitchat_agree",
"category": "generation_consistency",
"input": "今天天气不错",
"expect_branch": "fallback",
"generator_commands": [
{
"kind": "chitchat"
}
]
}
],
"guard_domains": [
"guard"
],
"protected_domains": [
"guard"
],
"builtin_specs": [
{
"id": "guard.config_change",
"domain": "guard",
"version": 1,
"priority": 1000,
"matcher": {
"includes_any": [
"系统配置",
"网关",
"改密码"
]
},
"risk": "high",
"handler": "refuse.and_guide"
}
]
}
+381
View File
@@ -0,0 +1,381 @@
package intent
import (
"context"
"math"
"testing"
)
// TestU5RegisterMutationDoesNotAffectSnapshot:登记后改原 spec 不影响 SpecByID/Catalog。
func TestU5RegisterMutationDoesNotAffectSnapshot(t *testing.T) {
spec := IntentSpec{
ID: "demo.iso.reg", Domain: "demo", Version: 1,
Matcher: MatchSpec{IncludesAny: []string{"隔离登记"}},
Risk: RiskHigh,
RequiresConfirm: true,
Handler: "iso.handler",
Examples: []string{"示例甲"},
Action: &ActionRef{Kind: ActionWorkflow, ID: "wf-iso"},
Slots: []SlotSpec{{
Key: "env", Pattern: `env[: ]*(\w+)`, Group: 1,
Enum: []string{"prod", "dev"}, Normalize: []string{"trim"},
}},
Metadata: map[string]string{"guidance": "keep", "token": "secret"},
}
b := NewRegistryBuilder(Options{ProtectedDomains: []string{"demo"}, GuardDomains: []string{"guard"}})
if err := b.Register(spec); err != nil {
t.Fatal(err)
}
// 登记后篡改原对象与嵌套可变字段。
spec.Risk = RiskLow
spec.RequiresConfirm = false
spec.Handler = "tampered"
spec.Examples[0] = "被改"
spec.Action.ID = "wf-tampered"
spec.Slots[0].Enum[0] = "evil"
spec.Metadata["guidance"] = "mutated"
spec.Matcher.IncludesAny[0] = "被改词"
reg, err := b.Build()
if err != nil {
t.Fatal(err)
}
snap := reg.Snapshot()
got, ok := snap.SpecByID("demo.iso.reg")
if !ok {
t.Fatal("missing spec")
}
if got.Risk != RiskHigh || !got.RequiresConfirm || got.Handler != "iso.handler" {
t.Fatalf("register isolation broken: %+v", got)
}
if got.Examples[0] != "示例甲" || got.Action == nil || got.Action.ID != "wf-iso" {
t.Fatalf("nested isolation broken: examples=%v action=%+v", got.Examples, got.Action)
}
if got.Slots[0].Enum[0] != "prod" || got.Metadata["guidance"] != "keep" {
t.Fatalf("slot/meta isolation broken: %+v", got)
}
cat := snap.Catalog()
if len(cat) != 1 || cat[0].Handler != "iso.handler" || cat[0].Slots[0].Enum[0] != "prod" {
t.Fatalf("catalog isolation after register mutate: %+v", cat)
}
opts := reg.Options()
opts.ProtectedDomains[0] = "hacked"
opts2 := reg.Options()
if opts2.ProtectedDomains[0] != "demo" {
t.Fatalf("Options isolation broken: %v", opts2.ProtectedDomains)
}
}
// TestU5ReturnedSpecMutationDoesNotAffectRegistry:改返回值不影响快照。
func TestU5ReturnedSpecMutationDoesNotAffectRegistry(t *testing.T) {
b := NewRegistryBuilder(Options{})
if err := b.Register(IntentSpec{
ID: "demo.iso.ret", Domain: "demo", Version: 1,
Matcher: MatchSpec{IncludesAny: []string{"隔离返回"}, Patterns: []string{`隔离返回`}},
Examples: []string{"例"},
Action: &ActionRef{Kind: ActionCommand, ID: "cmd.iso"},
Slots: []SlotSpec{{Key: "x", Pattern: `(x)`, Enum: []string{"a", "b"}}},
Metadata: map[string]string{"guidance": "g"},
}); err != nil {
t.Fatal(err)
}
reg, err := b.Build()
if err != nil {
t.Fatal(err)
}
snap := reg.Snapshot()
got, _ := snap.SpecByID("demo.iso.ret")
got.Examples[0] = "mut"
got.Action.ID = "cmd.mut"
got.Slots[0].Enum[0] = "z"
got.Metadata["guidance"] = "mut"
got.Matcher.Patterns[0] = "mut"
again, _ := snap.SpecByID("demo.iso.ret")
if again.Examples[0] != "例" || again.Action.ID != "cmd.iso" || again.Slots[0].Enum[0] != "a" {
t.Fatalf("SpecByID leak: %+v", again)
}
if again.Metadata["guidance"] != "g" || again.Matcher.Patterns[0] != `隔离返回` {
t.Fatalf("SpecByID matcher/meta leak: %+v", again)
}
cat := snap.Catalog()
cat[0].Slots[0].Enum[0] = "z"
cat[0].Metadata["guidance"] = "mut"
cat2 := snap.Catalog()
if cat2[0].Slots[0].Enum[0] != "a" || cat2[0].Metadata["guidance"] != "g" {
t.Fatalf("Catalog leak: %+v", cat2)
}
}
// TestU5GeneratedMatchesRuleValidationAndMetadata:同 action+slots 时生成路径与规则路径校验/元数据一致。
func TestU5GeneratedMatchesRuleValidationAndMetadata(t *testing.T) {
spec := IntentSpec{
ID: "report.generate", Domain: "report", Version: 1,
Matcher: MatchSpec{IncludesAny: []string{"生成报告"}},
Slots: []SlotSpec{{
Key: "topic", Required: true, Pattern: `主题是(.+)`, Group: 1,
Enum: []string{"电机", "泵"}, Normalize: []string{"trim"},
}},
Risk: RiskMedium,
RequiresConfirm: true,
Handler: "reportHandler",
Action: &ActionRef{Kind: ActionWorkflow, ID: "wf-report"},
Metadata: map[string]string{"guidance": "ask topic", "blueprint": "bp"},
}
b := NewRegistryBuilder(Options{})
if err := b.Register(spec); err != nil {
t.Fatal(err)
}
reg, err := b.Build()
if err != nil {
t.Fatal(err)
}
snap := reg.Snapshot()
rec := NewRecognizer(Options{}, nil, nil)
rule, err := rec.Recognize(context.Background(), "生成报告 主题是电机", snap)
if err != nil {
t.Fatal(err)
}
if rule.Branch != BranchMatched || rule.Top == nil {
t.Fatalf("rule: %+v", rule)
}
gen := stubGenerator{cmds: []Command{
{Kind: CommandStart, Action: ActionRef{Kind: ActionWorkflow, ID: "wf-report"},
Slots: map[string]string{"topic": "电机"}},
}}
genRec := NewRecognizer(Options{Generator: gen}, nil, nil)
out, err := genRec.Recognize(context.Background(), "完全不相关的一句话", snap)
if err != nil {
t.Fatal(err)
}
if out.Branch != BranchMatched || out.Top == nil {
t.Fatalf("generated: %+v", out)
}
if out.Top.ID != rule.Top.ID || out.Top.Risk != rule.Top.Risk || out.Top.RequiresConfirm != rule.Top.RequiresConfirm {
t.Fatalf("metadata mismatch rule=%+v gen=%+v", rule.Top, out.Top)
}
if out.Top.Handler != rule.Top.Handler {
t.Fatalf("handler mismatch rule=%q gen=%q", rule.Top.Handler, out.Top.Handler)
}
if out.Top.Action == nil || rule.Top.Action == nil || out.Top.Action.Kind != rule.Top.Action.Kind || out.Top.Action.ID != rule.Top.Action.ID {
t.Fatalf("action mismatch rule=%+v gen=%+v", rule.Top.Action, out.Top.Action)
}
if out.Top.Slots["topic"] != rule.Top.Slots["topic"] {
t.Fatalf("slots mismatch rule=%v gen=%v", rule.Top.Slots, out.Top.Slots)
}
if len(out.Top.MissingRequiredSlots) != 0 || len(out.Top.InvalidSlots) != 0 {
t.Fatalf("unexpected missing/invalid: %+v", out.Top)
}
}
// TestU5GeneratedSlotValidationStable:缺必填/非法枚举/未知键/重复 handler 在生成路径稳定处理。
func TestU5GeneratedSlotValidationStable(t *testing.T) {
mk := func(t *testing.T, specs ...IntentSpec) *Snapshot {
t.Helper()
b := NewRegistryBuilder(Options{})
if err := b.Register(specs...); err != nil {
t.Fatal(err)
}
reg, err := b.Build()
if err != nil {
t.Fatal(err)
}
return reg.Snapshot()
}
base := IntentSpec{
ID: "report.generate", Domain: "report", Version: 1,
Matcher: MatchSpec{IncludesAny: []string{"生成报告"}},
Slots: []SlotSpec{{
Key: "topic", Required: true, Pattern: `主题是(.+)`,
Enum: []string{"电机", "泵"},
}},
Risk: RiskHigh, RequiresConfirm: true, Handler: "reportHandler",
Action: &ActionRef{Kind: ActionCommand, ID: "cmd.report"},
}
t.Run("missing_required", func(t *testing.T) {
snap := mk(t, base)
rec := NewRecognizer(Options{Generator: stubGenerator{cmds: []Command{
{Kind: CommandStart, Action: ActionRef{Kind: ActionCommand, ID: "cmd.report"}, Slots: map[string]string{}},
}}}, nil, nil)
res, err := rec.Recognize(context.Background(), "无关输入", snap)
if err != nil {
t.Fatal(err)
}
if res.Branch != BranchMatched || res.Top == nil {
t.Fatalf("want matched with missing: %+v", res)
}
if len(res.Top.MissingRequiredSlots) != 1 || res.Top.MissingRequiredSlots[0] != "topic" {
t.Fatalf("missing: %+v", res.Top.MissingRequiredSlots)
}
if res.Top.Risk != RiskHigh || !res.Top.RequiresConfirm {
t.Fatalf("metadata lost: %+v", res.Top)
}
})
t.Run("invalid_enum", func(t *testing.T) {
snap := mk(t, base)
rec := NewRecognizer(Options{Generator: stubGenerator{cmds: []Command{
{Kind: CommandStart, Action: ActionRef{Kind: ActionHandler, ID: "reportHandler"},
Slots: map[string]string{"topic": "涡轮"}},
}}}, nil, nil)
res, err := rec.Recognize(context.Background(), "无关输入", snap)
if err != nil {
t.Fatal(err)
}
if res.Branch != BranchMatched || res.Top == nil {
t.Fatalf("want matched with invalid: %+v", res)
}
if res.Top.InvalidSlots["topic"] == "" {
t.Fatalf("invalid slots: %+v", res.Top.InvalidSlots)
}
if res.Top.Slots["topic"] != "" {
t.Fatalf("invalid value must not fill slots: %+v", res.Top.Slots)
}
})
t.Run("unknown_slot", func(t *testing.T) {
snap := mk(t, base)
rec := NewRecognizer(Options{Generator: stubGenerator{cmds: []Command{
{Kind: CommandStart, Action: ActionRef{Kind: ActionWorkflow, ID: "report.generate"},
Slots: map[string]string{"bogus": "v"}},
}}}, nil, nil)
res, err := rec.Recognize(context.Background(), "无关输入", snap)
if err != nil {
t.Fatal(err)
}
if res.Branch != BranchFallback || res.Top != nil {
t.Fatalf("unknown slot must fallback: %+v", res)
}
})
t.Run("duplicate_handler", func(t *testing.T) {
a := base
a.ID = "report.a"
a.Handler = "shared"
bSpec := base
bSpec.ID = "report.b"
bSpec.Handler = "shared"
bSpec.Action = &ActionRef{Kind: ActionCommand, ID: "cmd.b"}
snap := mk(t, a, bSpec)
rec := NewRecognizer(Options{Generator: stubGenerator{cmds: []Command{
{Kind: CommandStart, Action: ActionRef{Kind: ActionHandler, ID: "shared"},
Slots: map[string]string{"topic": "电机"}},
}}}, nil, nil)
res, err := rec.Recognize(context.Background(), "无关输入", snap)
if err != nil {
t.Fatal(err)
}
if res.Branch != BranchFallback || res.Top != nil {
t.Fatalf("ambiguous handler must fallback: %+v", res)
}
})
}
// TestU5GeneratedActionAliasRoutesToSpec:Action 指向不同 command ID 时仍解析到权威 spec。
func TestU5GeneratedActionAliasRoutesToSpec(t *testing.T) {
b := NewRegistryBuilder(Options{})
if err := b.Register(IntentSpec{
ID: "report.generate", Domain: "report", Version: 1,
Matcher: MatchSpec{IncludesAny: []string{"生成报告"}},
Slots: []SlotSpec{{Key: "topic", Pattern: `(.+)`}},
Action: &ActionRef{Kind: ActionCommand, ID: "cmd.other-id"},
Handler: "h",
Risk: RiskLow,
}); err != nil {
t.Fatal(err)
}
reg, err := b.Build()
if err != nil {
t.Fatal(err)
}
rec := NewRecognizer(Options{Generator: stubGenerator{cmds: []Command{
{Kind: CommandStart, Action: ActionRef{Kind: ActionCommand, ID: "cmd.other-id"},
Slots: map[string]string{"topic": "x"}},
}}}, nil, nil)
res, err := rec.Recognize(context.Background(), "无关输入", reg.Snapshot())
if err != nil {
t.Fatal(err)
}
if res.Branch != BranchMatched || res.Top == nil || res.Top.ID != "report.generate" {
t.Fatalf("alias resolve: %+v", res)
}
if res.Top.Action == nil || res.Top.Action.ID != "cmd.other-id" {
t.Fatalf("action metadata: %+v", res.Top.Action)
}
}
// TestU5ClassifierAndSemanticRejectBadScores:非有限/越界分数不得变成 matched。
func TestU5ClassifierAndSemanticRejectBadScores(t *testing.T) {
snap := ambiguousFixture(t, []string{"查运行进度"}, []string{"闲聊问候"})
t.Run("classifier_inf", func(t *testing.T) {
rec := NewRecognizer(Options{}, nil, rewriteClassifier{fn: func(cands []Candidate) []Candidate {
out := append([]Candidate(nil), cands...)
out[0].Confidence = math.Inf(1)
out[1].Confidence = 0.1
return out
}})
res, err := rec.Recognize(context.Background(), "模糊词", snap)
if err != nil {
t.Fatal(err)
}
if res.Branch != BranchAmbiguous || res.Top != nil {
t.Fatalf("inf must stay ambiguous: %+v", res)
}
})
t.Run("classifier_in_place_mutation", func(t *testing.T) {
rec := NewRecognizer(Options{}, nil, rewriteClassifier{fn: func(cands []Candidate) []Candidate {
cands[0].Handler = "tampered"
cands[0].Confidence = 0.99
cands[1].Confidence = 0.1
return cands
}})
res, err := rec.Recognize(context.Background(), "模糊词", snap)
if err != nil {
t.Fatal(err)
}
if res.Branch != BranchMatched || res.Top == nil {
t.Fatalf("expected matched: %+v", res)
}
if res.Top.Handler == "tampered" {
t.Fatalf("classifier in-place mutation leaked into top: %+v", res.Top)
}
})
t.Run("semantic_nan", func(t *testing.T) {
rec := NewRecognizer(Options{Semantic: fakeScorer{scores: map[string]float64{"查运行进度": math.NaN()}}}, nil, nil)
res, err := rec.Recognize(context.Background(), "模糊词", snap)
if err != nil {
t.Fatal(err)
}
if res.Branch != BranchAmbiguous || res.Top != nil {
t.Fatalf("nan semantic must stay ambiguous: %+v", res)
}
})
t.Run("semantic_above_one", func(t *testing.T) {
rec := NewRecognizer(Options{Semantic: fakeScorer{scores: map[string]float64{"查运行进度": 1.5}}}, nil, nil)
res, err := rec.Recognize(context.Background(), "模糊词", snap)
if err != nil {
t.Fatal(err)
}
if res.Branch != BranchAmbiguous || res.Top != nil {
t.Fatalf("out-of-range semantic must stay ambiguous: %+v", res)
}
})
t.Run("semantic_inf_not_matched", func(t *testing.T) {
rec := NewRecognizer(Options{Semantic: fakeScorer{scores: map[string]float64{"查运行进度": math.Inf(1)}}}, nil, nil)
res, err := rec.Recognize(context.Background(), "模糊词", snap)
if err != nil {
t.Fatal(err)
}
if res.Branch != BranchAmbiguous || res.Top != nil {
t.Fatalf("inf semantic must stay ambiguous: %+v", res)
}
})
}