Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7f3ff6a187 | ||
|
|
6aa4925301 |
+39
-6
@@ -1,5 +1,6 @@
|
||||
// intentctl 是意图配置的只读诊断 CLI:lint 校验配置并报告冲突,
|
||||
// explain 输出单条输入的裁决轨迹,eval 跑评测集。所有命令不改配置、不写文件。
|
||||
// explain 输出单条输入的裁决轨迹,eval 跑评测集。所有命令不改配置;
|
||||
// 唯一的写文件入口是 eval 显式给出的 -metrics-out(指标 JSON)。
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -42,7 +43,7 @@ func run(args []string) error {
|
||||
}
|
||||
|
||||
func usage() error {
|
||||
return fmt.Errorf("用法:intentctl <lint|explain|eval|catalog> ...\n lint -root <dir> [-protected a,b] [-guard a,b]\n explain -root <dir> [-protected a,b] [-guard a,b] <input>\n eval -root <dir> [-protected a,b] [-guard a,b] <corpus.json>\n catalog -root <dir> [-protected a,b] [-guard a,b] 输出能力目录(注入规划者/领导提示词用)")
|
||||
return fmt.Errorf("用法:intentctl <lint|explain|eval|catalog> ...\n lint -root <dir> [-protected a,b] [-guard a,b]\n explain -root <dir> [-protected a,b] [-guard a,b] <input>\n eval -root <dir> [-protected a,b] [-guard a,b] [-metrics-out <file|->] <corpus.json>\n catalog -root <dir> [-protected a,b] [-guard a,b] 输出能力目录(注入规划者/领导提示词用)")
|
||||
}
|
||||
|
||||
type commonOpts struct {
|
||||
@@ -243,16 +244,20 @@ func sortedKeys(m map[string]string) []string {
|
||||
return keys
|
||||
}
|
||||
|
||||
// evalCorpus 的 record_only 声明语料为基线评测集:期望描述理想行为,
|
||||
// 差异照常打印并计入指标,但不使 eval 失败(不设质量阈值)。
|
||||
type evalCorpus struct {
|
||||
Cases []intent.CorpusCase `json:"cases"`
|
||||
BuiltinSpecs []intent.IntentSpec `json:"builtin_specs,omitempty"`
|
||||
GuardDomains []string `json:"guard_domains,omitempty"`
|
||||
ProtectedDomains []string `json:"protected_domains,omitempty"`
|
||||
Cases []intent.CorpusCase `json:"cases"`
|
||||
BuiltinSpecs []intent.IntentSpec `json:"builtin_specs,omitempty"`
|
||||
GuardDomains []string `json:"guard_domains,omitempty"`
|
||||
ProtectedDomains []string `json:"protected_domains,omitempty"`
|
||||
RecordOnly bool `json:"record_only,omitempty"`
|
||||
}
|
||||
|
||||
func cmdEval(args []string) error {
|
||||
fs := flag.NewFlagSet("eval", flag.ContinueOnError)
|
||||
c := parseCommon(fs)
|
||||
metricsOut := fs.String("metrics-out", "", "指标 JSON 输出路径(- 为标准输出;留空不输出)")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -356,8 +361,36 @@ func cmdEval(args []string) error {
|
||||
rs = fmt.Sprintf("%.3f", r)
|
||||
}
|
||||
fmt.Printf("指标: precision=%s recall=%s(TP=%d FP=%d FN=%d)\n", ps, rs, metrics.TP, metrics.FP, metrics.FN)
|
||||
summary := intent.SummarizeCorpus(intent.CorpusDigest(data), corpus.Cases, results)
|
||||
fmt.Print(summary.Table())
|
||||
if *metricsOut != "" {
|
||||
if err := writeMetrics(*metricsOut, summary); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if failed > 0 {
|
||||
if corpus.RecordOnly {
|
||||
fmt.Printf("基线语料(record_only):%d/%d 用例与理想期望不符,只记录不判失败\n", failed, len(corpus.Cases))
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%d/%d 用例失败", failed, len(corpus.Cases))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeMetrics(path string, summary intent.EvalSummary) error {
|
||||
out, err := json.MarshalIndent(summary, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out = append(out, '\n')
|
||||
if path == "-" {
|
||||
_, err = os.Stdout.Write(out)
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(path, out, 0o644); err != nil {
|
||||
return fmt.Errorf("写指标文件失败:%w", err)
|
||||
}
|
||||
fmt.Printf("指标 JSON 已写入 %s\n", path)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,9 +1,13 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"agent-intent/intent"
|
||||
)
|
||||
|
||||
const testSpecJSON = `{
|
||||
@@ -71,3 +75,39 @@ func TestEvalFailsOnWrongExpectation(t *testing.T) {
|
||||
t.Fatal("expected eval failure on wrong expectation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvalRecordOnlyWritesMetrics(t *testing.T) {
|
||||
dir := writeTemp(t, "specs.json", testSpecJSON)
|
||||
tmp := t.TempDir()
|
||||
corpus := filepath.Join(tmp, "corpus.json")
|
||||
// 「不用查进度了」在否定修复后已正确回落,这里换成仍会误执行的输入来覆盖 record_only 计数。
|
||||
doc := `{"record_only": true, "cases": [
|
||||
{"name": "miss", "category": "negation_reference", "input": "进度条的颜色真好看", "expect_branch": "fallback"},
|
||||
{"name": "hit", "input": "天气不错", "expect_branch": "fallback"}]}`
|
||||
if err := os.WriteFile(corpus, []byte(doc), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := filepath.Join(tmp, "metrics.json")
|
||||
if err := run([]string{"eval", "-root", dir, "-protected", "", "-metrics-out", out, corpus}); err != nil {
|
||||
t.Fatalf("record_only eval should not fail: %v", err)
|
||||
}
|
||||
data, err := os.ReadFile(out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var s intent.EvalSummary
|
||||
if err := json.Unmarshal(data, &s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s.Schema != intent.EvalMetricsSchema || s.Overall.Cases != 2 || len(s.Categories) != 2 || s.Overall.MisExecutions != 1 {
|
||||
t.Fatalf("summary = %+v", s)
|
||||
}
|
||||
|
||||
strict := filepath.Join(tmp, "strict.json")
|
||||
if err := os.WriteFile(strict, []byte(strings.Replace(doc, `"record_only": true, `, "", 1)), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := run([]string{"eval", "-root", dir, "-protected", "", strict}); err == nil {
|
||||
t.Fatal("corpus without record_only must still fail on mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
# 能力召回评测基线(A3)
|
||||
|
||||
本文记录召回评测集在当前识别器上的**指标基线**。只记录,不设质量阈值、不做通过判定;
|
||||
据此决定后续方向(检索、模型调优等)不在本基线范围内。机器可读基线为同目录
|
||||
[`metrics.json`](metrics.json)(schema `agent-intent.eval-metrics/v1`),本表数字与之一致。
|
||||
|
||||
## 语料
|
||||
|
||||
- 位置:[`intent/testdata/recall_corpus.json`](../intent/testdata/recall_corpus.json),
|
||||
与 `intent/testdata/corpus.json`、`corpus_multiturn.json` 同一格式(`CorpusCase`),
|
||||
由 `intentctl eval` 与 `go test` 共用。
|
||||
- 摘要:`sha256:53d26cfc4caad994074db7d49924e3eabcba7c8ada4d6389a9a23f3085bc2885`
|
||||
(语料文件原始字节的 sha256;语料变更后须重跑并更新本文与 `metrics.json`)。
|
||||
- 数据:全部为合成用例,只复用测试夹具 `intent/testdata/specs/base.json` 的域、动作与槽位形态,
|
||||
以及语料自声明的守卫 spec `guard.config_change`;不含真实用户数据。
|
||||
- 期望写的是**理想行为**(不是当前行为的快照),语料声明 `record_only: true`:
|
||||
与期望不符的用例照常打印 FAIL 并计入指标,但不使 eval 失败。
|
||||
- 离线:生成路径用确定性回放生成器(回放用例 `generator_commands`),无模型、无网络调用。
|
||||
|
||||
### 类别覆盖
|
||||
|
||||
| 类别 | 用例 | 轮次 | 覆盖内容 |
|
||||
|---|---|---|---|
|
||||
| `negation_reference` | 8 | 10 | 否定指令(不要/别/不用/先不要)、`excludes_any` 否定、「还是刚才那个」类指代(单轮无上下文与多轮) |
|
||||
| `multi_intent` | 7 | 7 | 一句含两个意图信号:动作+动作(应澄清)、寒暄+动作(应取动作)、动作+守卫(应守卫拦截) |
|
||||
| `wrong_slot` | 7 | 7 | 槽位缺失、中文数字、两个候选值、越出字符集、仅符号、空主题 |
|
||||
| `expired_confirmation` | 7 | 14 | 上一轮挂起预期(链式或宿主声明)在下一轮到达前过期:裸回答应回落、完整新指令应走规则 matched 而非 expectation_hit;含 1 例未过期对照 |
|
||||
| `generation_consistency` | 8 | 8 | 同一输入同时走规则路径与生成路径(回放生成器 + 识别器兜底同款校验),比对 Top 与槽位;含槽位分歧、错动作、幻觉动作、闲聊 |
|
||||
| 合计 | 37 | 46 | |
|
||||
|
||||
## 指标口径
|
||||
|
||||
按轮次计(v1 单轮用例即一轮);比率保留 4 位,分母为 0 记 `-`(JSON `null`)。
|
||||
|
||||
- **pass**:全部轮次与理想期望一致的用例数。
|
||||
- **top1**:写了 `expect_top` 的轮次中,实际 Top 等于期望的比例。
|
||||
- **mis_exec(误执行)**:实际 `matched`/`expectation_hit` 出了 Top,但与期望动作不同
|
||||
(期望别的 Top,或期望 `fallback`/`ambiguous`/`guard_hit` 却执行了)——危险项。
|
||||
- **clarify(澄清率)**:`guard_hit` 或产出 `clarify` 命令的轮次占比。
|
||||
- **fallback(回落率)**:实际 `fallback` 的轮次占比。
|
||||
- **gen_agree(生成一致率)**:`generation_consistency` 用例中规则路径与生成路径
|
||||
Top ID 与槽位都一致的比例(两边都无 Top 也算一致)。
|
||||
|
||||
## 基线(本次运行)
|
||||
|
||||
| 类别 | 用例 | pass | 轮次 | top1 | mis_exec | clarify | fallback | gen_agree |
|
||||
|---|---|---|---|---|---|---|---|---|
|
||||
| `expired_confirmation` | 7 | 7 | 14 | 11/11 = 1.0000 | 0 | 5/14 = 0.3571 | 3/14 = 0.2143 | - |
|
||||
| `generation_consistency` | 8 | 8 | 8 | 7/7 = 1.0000 | 0 | 0/8 = 0.0000 | 1/8 = 0.1250 | 5/8 = 0.6250 |
|
||||
| `multi_intent` | 7 | 7 | 7 | 3/3 = 1.0000 | 0 | 5/7 = 0.7143 | 0/7 = 0.0000 | - |
|
||||
| `negation_reference` | 8 | 8 | 10 | 3/3 = 1.0000 | 0 | 0/10 = 0.0000 | 7/10 = 0.7000 | - |
|
||||
| `wrong_slot` | 7 | 7 | 7 | 7/7 = 1.0000 | 0 | 7/7 = 1.0000 | 0/7 = 0.0000 | - |
|
||||
| **overall** | 37 | 37 | 46 | 31/31 = 1.0000 | 0 | 17/46 = 0.3696 | 11/46 = 0.2391 | 5/8 = 0.6250 |
|
||||
|
||||
同次运行的既有 I2 指标:precision=1.000 recall=1.000(TP=30 FP=0 FN=0)。
|
||||
|
||||
首次基线(规则层修复前)为 pass 27/37、top1 30/31、mis_exec 6、clarify 13/46、fallback 6/46,
|
||||
差异集中在 `negation_reference`(5 次误执行)、`multi_intent`(1 次误执行)、`wrong_slot`(4 例未澄清)。
|
||||
语料未变(摘要同上),上表变化全部来自规则层代码修复。
|
||||
|
||||
### 记录到的差异(事实,不含判定)
|
||||
|
||||
- `negation_reference`:「不要重启任务」「别取消任务 42」「不用查进度了」「先不要发布」「刚才那个任务先别重启」
|
||||
均回落——词条/pattern 命中前紧邻或相隔至多 2 个字符出现否定词(不要、别、不用、先不要、先别、无需、不需要)时,
|
||||
该次命中不计正向,全部命中被否定的 spec 以 `negated_match` 排除;`excludes_any`(「配置」)照旧整体排除。
|
||||
- `multi_intent`:「你好,帮我生成报告,主题是电机」裁决到 `report.generate`——同分组内抽到槽位更多的候选
|
||||
胜过 priority 不同但未抽到槽位的寒暄;priority 相同的两个动作并列时仍进入 ambiguous 澄清。
|
||||
- `wrong_slot`:两个候选值(「任务 42 和任务 43」「任务 42 还是任务 43」)、越出字符集(`abc_def` 被截成 `abc`)、
|
||||
仅符号(`--`)均判为非法槽位,产出 start + clarify;缺失与中文数字仍进入 clarify。
|
||||
- `expired_confirmation`:过期后裸回答回落、完整指令走规则 matched,均与期望一致。
|
||||
- `generation_consistency`:3 例不一致——生成槽位 `run_id=43` 与规则 `42` 不同、生成错动作
|
||||
`taskrun.restart`(规则为 status_query)、幻觉动作 `ghost.flow` 被校验 fail-closed 后无 Top。
|
||||
|
||||
## 重跑
|
||||
|
||||
在仓库根(依赖子模块已按 gitlink 初始化):
|
||||
|
||||
```bash
|
||||
# 打印逐例 PASS/FAIL、指标表,并写机器基线(-metrics-out - 则输出到标准输出)
|
||||
GOWORK=off go run ./cmd/intentctl eval -root intent/testdata/specs \
|
||||
-metrics-out evaluation/metrics.json intent/testdata/recall_corpus.json
|
||||
|
||||
# 结构覆盖测试(≥30 例、五类齐全、每例可执行、指标 JSON 合法且与计数一致、基线摘要与语料一致)
|
||||
GOWORK=off go test -count=1 -run 'TestRecallCorpusStructure' -v ./intent/
|
||||
```
|
||||
|
||||
`-root` 指向测试同款夹具目录;守卫 spec 与守卫/保护域由语料 `builtin_specs`、`guard_domains`、
|
||||
`protected_domains` 自声明,测试按同样方式建快照,因此两条命令的指标同源。
|
||||
识别是确定性的:同语料、同代码重跑得到逐字节相同的 `metrics.json`。
|
||||
@@ -0,0 +1,108 @@
|
||||
{
|
||||
"schema": "agent-intent.eval-metrics/v1",
|
||||
"corpus_digest": "sha256:53d26cfc4caad994074db7d49924e3eabcba7c8ada4d6389a9a23f3085bc2885",
|
||||
"overall": {
|
||||
"category": "overall",
|
||||
"cases": 37,
|
||||
"passed_cases": 37,
|
||||
"turns": 46,
|
||||
"top1_expected": 31,
|
||||
"top1_hits": 31,
|
||||
"top1_hit_rate": 1,
|
||||
"mis_executions": 0,
|
||||
"clarifications": 17,
|
||||
"clarification_rate": 0.3696,
|
||||
"fallbacks": 11,
|
||||
"fallback_rate": 0.2391,
|
||||
"generation_cases": 8,
|
||||
"generation_agree": 5,
|
||||
"generation_agreement_rate": 0.625
|
||||
},
|
||||
"categories": [
|
||||
{
|
||||
"category": "expired_confirmation",
|
||||
"cases": 7,
|
||||
"passed_cases": 7,
|
||||
"turns": 14,
|
||||
"top1_expected": 11,
|
||||
"top1_hits": 11,
|
||||
"top1_hit_rate": 1,
|
||||
"mis_executions": 0,
|
||||
"clarifications": 5,
|
||||
"clarification_rate": 0.3571,
|
||||
"fallbacks": 3,
|
||||
"fallback_rate": 0.2143,
|
||||
"generation_cases": 0,
|
||||
"generation_agree": 0,
|
||||
"generation_agreement_rate": null
|
||||
},
|
||||
{
|
||||
"category": "generation_consistency",
|
||||
"cases": 8,
|
||||
"passed_cases": 8,
|
||||
"turns": 8,
|
||||
"top1_expected": 7,
|
||||
"top1_hits": 7,
|
||||
"top1_hit_rate": 1,
|
||||
"mis_executions": 0,
|
||||
"clarifications": 0,
|
||||
"clarification_rate": 0,
|
||||
"fallbacks": 1,
|
||||
"fallback_rate": 0.125,
|
||||
"generation_cases": 8,
|
||||
"generation_agree": 5,
|
||||
"generation_agreement_rate": 0.625
|
||||
},
|
||||
{
|
||||
"category": "multi_intent",
|
||||
"cases": 7,
|
||||
"passed_cases": 7,
|
||||
"turns": 7,
|
||||
"top1_expected": 3,
|
||||
"top1_hits": 3,
|
||||
"top1_hit_rate": 1,
|
||||
"mis_executions": 0,
|
||||
"clarifications": 5,
|
||||
"clarification_rate": 0.7143,
|
||||
"fallbacks": 0,
|
||||
"fallback_rate": 0,
|
||||
"generation_cases": 0,
|
||||
"generation_agree": 0,
|
||||
"generation_agreement_rate": null
|
||||
},
|
||||
{
|
||||
"category": "negation_reference",
|
||||
"cases": 8,
|
||||
"passed_cases": 8,
|
||||
"turns": 10,
|
||||
"top1_expected": 3,
|
||||
"top1_hits": 3,
|
||||
"top1_hit_rate": 1,
|
||||
"mis_executions": 0,
|
||||
"clarifications": 0,
|
||||
"clarification_rate": 0,
|
||||
"fallbacks": 7,
|
||||
"fallback_rate": 0.7,
|
||||
"generation_cases": 0,
|
||||
"generation_agree": 0,
|
||||
"generation_agreement_rate": null
|
||||
},
|
||||
{
|
||||
"category": "wrong_slot",
|
||||
"cases": 7,
|
||||
"passed_cases": 7,
|
||||
"turns": 7,
|
||||
"top1_expected": 7,
|
||||
"top1_hits": 7,
|
||||
"top1_hit_rate": 1,
|
||||
"mis_executions": 0,
|
||||
"clarifications": 7,
|
||||
"clarification_rate": 1,
|
||||
"fallbacks": 0,
|
||||
"fallback_rate": 0,
|
||||
"generation_cases": 0,
|
||||
"generation_agree": 0,
|
||||
"generation_agreement_rate": null
|
||||
}
|
||||
]
|
||||
}
|
||||
+9
-5
@@ -16,12 +16,8 @@ func boundedContains(haystack, term string) bool {
|
||||
if term == "" {
|
||||
return false
|
||||
}
|
||||
first, _ := utf8.DecodeRuneInString(term)
|
||||
last, _ := utf8.DecodeLastRuneInString(term)
|
||||
checkLeft := isWord(first) && !isHan(first) // 仅拉丁开头的词条要求左界
|
||||
for i := strings.Index(haystack, term); i >= 0; {
|
||||
end := i + len(term)
|
||||
if (!checkLeft || leftBounded(haystack, i, first)) && rightBounded(haystack, end, last) {
|
||||
if boundedAt(haystack, i, term) {
|
||||
return true
|
||||
}
|
||||
j := strings.Index(haystack[i+1:], term)
|
||||
@@ -33,6 +29,14 @@ func boundedContains(haystack, term string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// boundedAt 报告 haystack[i:] 处的 term 出现是否满足 boundedContains 的词界规则。
|
||||
func boundedAt(haystack string, i int, term string) bool {
|
||||
first, _ := utf8.DecodeRuneInString(term)
|
||||
last, _ := utf8.DecodeLastRuneInString(term)
|
||||
checkLeft := isWord(first) && !isHan(first) // 仅拉丁开头的词条要求左界
|
||||
return (!checkLeft || leftBounded(haystack, i, first)) && rightBounded(haystack, i+len(term), last)
|
||||
}
|
||||
|
||||
func leftBounded(s string, i int, termFirst rune) bool {
|
||||
if i == 0 {
|
||||
return true
|
||||
|
||||
+116
-13
@@ -7,22 +7,33 @@ import (
|
||||
|
||||
// CorpusCase 是评测用例:v1 单轮(input/expect_*)与 v2 多轮(turns)共用。
|
||||
// v1 字段在 Turns 为空时生效,并自动等价于一个单 turn 用例(不含命令断言)。
|
||||
// Category 与 GeneratorCommands 为可选字段(A3 召回评测):旧语料不写即按
|
||||
// 未分类处理;GeneratorCommands 是确定性回放生成器的脚本输出,写了才在
|
||||
// 首轮输入上额外跑一遍生成路径并与规则路径比对。
|
||||
type CorpusCase struct {
|
||||
Name string `json:"name"`
|
||||
Category string `json:"category,omitempty"`
|
||||
Input string `json:"input,omitempty"`
|
||||
ExpectBranch string `json:"expect_branch,omitempty"`
|
||||
ExpectTop string `json:"expect_top,omitempty"`
|
||||
ExpectSlots map[string]string `json:"expect_slots,omitempty"`
|
||||
ExpectMissingSlots []string `json:"expect_missing_slots,omitempty"`
|
||||
Turns []CorpusTurn `json:"turns,omitempty"`
|
||||
GeneratorCommands []Command `json:"generator_commands,omitempty"`
|
||||
}
|
||||
|
||||
// CorpusTurn 是多轮用例中的一轮:输入与逐轮期望(分支/top/命令列表)。
|
||||
// CorpusTurn 是多轮用例中的一轮:输入与逐轮期望(分支/top/槽位/命令列表)。
|
||||
// PendingExpectation 声明本轮到达前宿主挂起的预期意图(覆盖上一轮的链式预期);
|
||||
// ExpectationExpired 表示本轮到达时挂起预期已过期(声明的或链式的都作废)。
|
||||
type CorpusTurn struct {
|
||||
Input string `json:"input"`
|
||||
ExpectBranch string `json:"expect_branch,omitempty"`
|
||||
ExpectTop string `json:"expect_top,omitempty"`
|
||||
ExpectCommands []CommandExpect `json:"expect_commands,omitempty"`
|
||||
Input string `json:"input"`
|
||||
PendingExpectation []string `json:"pending_expectation,omitempty"`
|
||||
ExpectationExpired bool `json:"expectation_expired,omitempty"`
|
||||
ExpectBranch string `json:"expect_branch,omitempty"`
|
||||
ExpectTop string `json:"expect_top,omitempty"`
|
||||
ExpectSlots map[string]string `json:"expect_slots,omitempty"`
|
||||
ExpectMissingSlots []string `json:"expect_missing_slots,omitempty"`
|
||||
ExpectCommands []CommandExpect `json:"expect_commands,omitempty"`
|
||||
}
|
||||
|
||||
// CommandExpect 是对一条命令的期望;只有写出的字段参与断言。
|
||||
@@ -44,14 +55,29 @@ type TurnResult struct {
|
||||
Input string
|
||||
Branch string
|
||||
TopID string
|
||||
Slots map[string]string
|
||||
Commands []Command
|
||||
Mismatches []string
|
||||
}
|
||||
|
||||
// GeneratorComparison 是同一输入的规则路径与生成路径(确定性回放生成器,
|
||||
// 经与识别器兜底同款的校验)的 Top 比对。Agree = Top ID 与槽位都一致
|
||||
// (两边都没有 Top 也算一致)。
|
||||
type GeneratorComparison struct {
|
||||
Input string
|
||||
RulesTopID string
|
||||
GeneratorTopID string
|
||||
RulesSlots map[string]string
|
||||
GeneratorSlots map[string]string
|
||||
Agree bool
|
||||
}
|
||||
|
||||
// CaseResult 一个用例的全部轮次结果。Passed = 所有轮次无差异。
|
||||
// Generator 仅在用例写了 generator_commands 时非 nil;比对结果不计入 Passed。
|
||||
type CaseResult struct {
|
||||
Name string
|
||||
Turns []TurnResult
|
||||
Name string
|
||||
Turns []TurnResult
|
||||
Generator *GeneratorComparison
|
||||
}
|
||||
|
||||
// Passed 报告用例是否全部轮次通过。
|
||||
@@ -74,17 +100,31 @@ func (c *chainedExpectation) Pending(ctx context.Context) *PendingExpectation {
|
||||
return c.next
|
||||
}
|
||||
|
||||
// RunCorpusCase 执行一个用例的全部轮次(v1 用例自动包装为单轮)。
|
||||
// 识别器按轮次构建在链式预期上,保证 SetSlot+Start 这类组合可在评测中到达。
|
||||
func RunCorpusCase(ctx context.Context, snap *Snapshot, tc CorpusCase) CaseResult {
|
||||
turns := tc.Turns
|
||||
if len(turns) == 0 {
|
||||
turns = []CorpusTurn{{Input: tc.Input, ExpectBranch: tc.ExpectBranch, ExpectTop: tc.ExpectTop}}
|
||||
// corpusTurns 返回用例的轮次(v1 用例自动包装为单轮,携带 v1 槽位期望)。
|
||||
func corpusTurns(tc CorpusCase) []CorpusTurn {
|
||||
if len(tc.Turns) > 0 {
|
||||
return tc.Turns
|
||||
}
|
||||
return []CorpusTurn{{Input: tc.Input, ExpectBranch: tc.ExpectBranch, ExpectTop: tc.ExpectTop,
|
||||
ExpectSlots: tc.ExpectSlots, ExpectMissingSlots: tc.ExpectMissingSlots}}
|
||||
}
|
||||
|
||||
// RunCorpusCase 执行一个用例的全部轮次(v1 用例自动包装为单轮)。
|
||||
// 识别器按轮次构建在链式预期上,保证 SetSlot+Start 这类组合可在评测中到达;
|
||||
// 轮次可显式声明挂起预期或声明其已过期。写了 generator_commands 的用例
|
||||
// 另在首轮输入上跑生成路径并与规则路径比对(离线、确定性回放)。
|
||||
func RunCorpusCase(ctx context.Context, snap *Snapshot, tc CorpusCase) CaseResult {
|
||||
turns := corpusTurns(tc)
|
||||
exp := &chainedExpectation{}
|
||||
rec := NewRecognizer(Options{}, exp, nil)
|
||||
out := CaseResult{Name: tc.Name}
|
||||
for i, turn := range turns {
|
||||
if len(turn.PendingExpectation) > 0 {
|
||||
exp.next = &PendingExpectation{IntentIDs: append([]string(nil), turn.PendingExpectation...), Reason: "eval declared"}
|
||||
}
|
||||
if turn.ExpectationExpired {
|
||||
exp.next = nil
|
||||
}
|
||||
tr := TurnResult{Index: i, Input: turn.Input}
|
||||
res, err := rec.Recognize(ctx, turn.Input, snap)
|
||||
if err != nil {
|
||||
@@ -95,6 +135,7 @@ func RunCorpusCase(ctx context.Context, snap *Snapshot, tc CorpusCase) CaseResul
|
||||
tr.Branch = string(res.Branch)
|
||||
if res.Top != nil {
|
||||
tr.TopID = res.Top.ID
|
||||
tr.Slots = res.Top.Slots
|
||||
}
|
||||
tr.Commands = res.Commands
|
||||
tr.Mismatches = matchTurn(turn, res)
|
||||
@@ -104,9 +145,52 @@ func RunCorpusCase(ctx context.Context, snap *Snapshot, tc CorpusCase) CaseResul
|
||||
exp.next = &PendingExpectation{IntentIDs: []string{res.Top.ID}, Reason: "eval chained"}
|
||||
}
|
||||
}
|
||||
if len(tc.GeneratorCommands) > 0 && len(out.Turns) > 0 {
|
||||
out.Generator = compareGeneratorPath(ctx, snap, out.Turns[0], tc.GeneratorCommands)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// replayGenerator 是评测用确定性生成器:忽略输入与目录,回放用例脚本中的命令
|
||||
// (无模型、无网络)。
|
||||
type replayGenerator struct {
|
||||
cmds []Command
|
||||
}
|
||||
|
||||
func (g replayGenerator) Generate(ctx context.Context, input string, catalog []CatalogSpec) ([]Command, error) {
|
||||
out := make([]Command, len(g.cmds))
|
||||
for i, c := range g.cmds {
|
||||
c.Slots = cloneStringMap(c.Slots)
|
||||
out[i] = c
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// compareGeneratorPath 绕过规则阶段,直接走识别器兜底同款的生成路径
|
||||
// (目录 top-k → 生成 → 校验 → 解析回权威 spec),再与规则路径首轮比对。
|
||||
func compareGeneratorPath(ctx context.Context, snap *Snapshot, rules TurnResult, cmds []Command) *GeneratorComparison {
|
||||
cmp := &GeneratorComparison{Input: rules.Input, RulesTopID: rules.TopID, RulesSlots: rules.Slots}
|
||||
rec := NewRecognizer(Options{Generator: replayGenerator{cmds: cmds}}, nil, nil)
|
||||
if res, ok := rec.generateCommands(ctx, rules.Input, snap); ok && res.Top != nil {
|
||||
cmp.GeneratorTopID = res.Top.ID
|
||||
cmp.GeneratorSlots = res.Top.Slots
|
||||
}
|
||||
cmp.Agree = cmp.RulesTopID == cmp.GeneratorTopID && sameSlots(cmp.RulesSlots, cmp.GeneratorSlots)
|
||||
return cmp
|
||||
}
|
||||
|
||||
func sameSlots(a, b map[string]string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for k, v := range a {
|
||||
if w, ok := b[k]; !ok || w != v {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func matchTurn(turn CorpusTurn, res Result) []string {
|
||||
var mm []string
|
||||
if turn.ExpectBranch != "" && string(res.Branch) != turn.ExpectBranch {
|
||||
@@ -121,6 +205,24 @@ func matchTurn(turn CorpusTurn, res Result) []string {
|
||||
mm = append(mm, fmt.Sprintf("top=%s(期望 %s)", got, turn.ExpectTop))
|
||||
}
|
||||
}
|
||||
for _, k := range sortedKeys(turn.ExpectSlots) {
|
||||
got := ""
|
||||
if res.Top != nil {
|
||||
got = res.Top.Slots[k]
|
||||
}
|
||||
if got != turn.ExpectSlots[k] {
|
||||
mm = append(mm, fmt.Sprintf("slots[%s]=%q(期望 %q)", k, got, turn.ExpectSlots[k]))
|
||||
}
|
||||
}
|
||||
if len(turn.ExpectMissingSlots) > 0 {
|
||||
var got []string
|
||||
if res.Top != nil {
|
||||
got = res.Top.MissingRequiredSlots
|
||||
}
|
||||
if fmt.Sprint(got) != fmt.Sprint(turn.ExpectMissingSlots) {
|
||||
mm = append(mm, fmt.Sprintf("missing=%v(期望 %v)", got, turn.ExpectMissingSlots))
|
||||
}
|
||||
}
|
||||
if turn.ExpectCommands != nil {
|
||||
mm = append(mm, matchCommands(turn.ExpectCommands, res.Commands)...)
|
||||
}
|
||||
@@ -208,6 +310,7 @@ func (m CorpusMetrics) Recall() (float64, bool) {
|
||||
// - FP:实际 matched/top=Y 但期望的标签不是 matched/top=Y
|
||||
// (期望别的 top、期望 fallback/ambiguous、或期望 matched 但 top 不同);
|
||||
// - FN:期望 matched/top=X 但实际没拿到它(branch 不对或 top 不同)。
|
||||
//
|
||||
// 只有写了 expect_top 的轮次进入指标(意图识别通行口径)。
|
||||
func MetricsOf(cases []CorpusCase, results []CaseResult) CorpusMetrics {
|
||||
var m CorpusMetrics
|
||||
|
||||
@@ -75,4 +75,16 @@ func TestEvalCorpus(t *testing.T) {
|
||||
_, snap := buildDemoRegistry(t)
|
||||
rec := NewRecognizer(Options{}, nil, nil)
|
||||
runCorpus(t, rec, snap, doc.Cases)
|
||||
|
||||
var shared struct {
|
||||
Cases []CorpusCase `json:"cases"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &shared); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var results []CaseResult
|
||||
for _, tc := range shared.Cases {
|
||||
results = append(results, RunCorpusCase(context.Background(), snap, tc))
|
||||
}
|
||||
t.Log("\n" + SummarizeCorpus(CorpusDigest(data), shared.Cases, results).Table())
|
||||
}
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
package intent
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// EvalMetricsSchema 是机器可读评测指标(EvalSummary JSON)的格式标识。
|
||||
const EvalMetricsSchema = "agent-intent.eval-metrics/v1"
|
||||
|
||||
// UncategorizedCategory 是未写 category 的用例归入的类别(旧语料兼容)。
|
||||
const UncategorizedCategory = "uncategorized"
|
||||
|
||||
// CategoryMetrics 是一个类别(或全量)的评测指标(A3 召回基线)。
|
||||
// 计数口径按轮次(v1 用例即一轮),比率的分母为 0 时为 nil(JSON null):
|
||||
// - top1:写了 expect_top 的轮次中实际 Top 等于期望的比例;
|
||||
// - mis_executions:实际 matched/expectation_hit 出 Top,但与期望动作不同
|
||||
// (期望别的 top,或期望 fallback/ambiguous/guard_hit 却执行了)——危险项;
|
||||
// - clarification:guard_hit 或产出 clarify 命令的轮次;
|
||||
// - fallback:实际 fallback 的轮次;
|
||||
// - generation:写了 generator_commands 的用例中,规则路径与生成路径一致的比例。
|
||||
type CategoryMetrics struct {
|
||||
Category string `json:"category"`
|
||||
Cases int `json:"cases"`
|
||||
PassedCases int `json:"passed_cases"`
|
||||
Turns int `json:"turns"`
|
||||
Top1Expected int `json:"top1_expected"`
|
||||
Top1Hits int `json:"top1_hits"`
|
||||
Top1HitRate *float64 `json:"top1_hit_rate"`
|
||||
MisExecutions int `json:"mis_executions"`
|
||||
Clarifications int `json:"clarifications"`
|
||||
ClarificationRate *float64 `json:"clarification_rate"`
|
||||
Fallbacks int `json:"fallbacks"`
|
||||
FallbackRate *float64 `json:"fallback_rate"`
|
||||
GenerationCases int `json:"generation_cases"`
|
||||
GenerationAgree int `json:"generation_agree"`
|
||||
GenerationAgreementRate *float64 `json:"generation_agreement_rate"`
|
||||
}
|
||||
|
||||
// EvalSummary 是一次评测的指标汇总:按类别(名称升序)与全量,附语料摘要。
|
||||
// 只记录基线,不含阈值与通过判定。
|
||||
type EvalSummary struct {
|
||||
Schema string `json:"schema"`
|
||||
CorpusDigest string `json:"corpus_digest"`
|
||||
Overall CategoryMetrics `json:"overall"`
|
||||
Categories []CategoryMetrics `json:"categories"`
|
||||
}
|
||||
|
||||
// CorpusDigest 返回语料文件原始字节的摘要(sha256:<hex>)。
|
||||
func CorpusDigest(data []byte) string {
|
||||
sum := sha256.Sum256(data)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// SummarizeCorpus 汇总评测指标。results 须与 cases 按下标一一对应
|
||||
// (即 RunCorpusCase 逐个执行 cases 的结果)。
|
||||
func SummarizeCorpus(digest string, cases []CorpusCase, results []CaseResult) EvalSummary {
|
||||
byCat := map[string]*CategoryMetrics{}
|
||||
overall := &CategoryMetrics{Category: "overall"}
|
||||
for i, tc := range cases {
|
||||
if i >= len(results) {
|
||||
break
|
||||
}
|
||||
cat := tc.Category
|
||||
if strings.TrimSpace(cat) == "" {
|
||||
cat = UncategorizedCategory
|
||||
}
|
||||
m, ok := byCat[cat]
|
||||
if !ok {
|
||||
m = &CategoryMetrics{Category: cat}
|
||||
byCat[cat] = m
|
||||
}
|
||||
accumulateCase(m, tc, results[i])
|
||||
accumulateCase(overall, tc, results[i])
|
||||
}
|
||||
out := EvalSummary{Schema: EvalMetricsSchema, CorpusDigest: digest, Categories: []CategoryMetrics{}}
|
||||
for _, m := range byCat {
|
||||
finishRates(m)
|
||||
out.Categories = append(out.Categories, *m)
|
||||
}
|
||||
sort.Slice(out.Categories, func(i, j int) bool { return out.Categories[i].Category < out.Categories[j].Category })
|
||||
finishRates(overall)
|
||||
out.Overall = *overall
|
||||
return out
|
||||
}
|
||||
|
||||
func accumulateCase(m *CategoryMetrics, tc CorpusCase, res CaseResult) {
|
||||
m.Cases++
|
||||
if res.Passed() {
|
||||
m.PassedCases++
|
||||
}
|
||||
turns := corpusTurns(tc)
|
||||
for i, tr := range res.Turns {
|
||||
if i >= len(turns) {
|
||||
break
|
||||
}
|
||||
want := turns[i]
|
||||
m.Turns++
|
||||
if want.ExpectTop != "" {
|
||||
m.Top1Expected++
|
||||
if tr.TopID == want.ExpectTop {
|
||||
m.Top1Hits++
|
||||
}
|
||||
}
|
||||
if misExecuted(want, tr) {
|
||||
m.MisExecutions++
|
||||
}
|
||||
if tr.Branch == string(BranchGuardHit) || hasClarify(tr.Commands) {
|
||||
m.Clarifications++
|
||||
}
|
||||
if tr.Branch == string(BranchFallback) {
|
||||
m.Fallbacks++
|
||||
}
|
||||
}
|
||||
if res.Generator != nil {
|
||||
m.GenerationCases++
|
||||
if res.Generator.Agree {
|
||||
m.GenerationAgree++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// misExecuted 报告该轮是否执行了与期望不同的动作(危险项)。
|
||||
// 期望 matched/expectation_hit 但未写 expect_top 的轮次不约束动作,不计。
|
||||
func misExecuted(want CorpusTurn, got TurnResult) bool {
|
||||
executed := got.TopID != "" && (got.Branch == string(BranchMatched) || got.Branch == string(BranchExpectationHit))
|
||||
if !executed {
|
||||
return false
|
||||
}
|
||||
if want.ExpectTop != "" {
|
||||
return got.TopID != want.ExpectTop
|
||||
}
|
||||
switch Branch(want.ExpectBranch) {
|
||||
case BranchFallback, BranchAmbiguous, BranchGuardHit:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func hasClarify(cmds []Command) bool {
|
||||
for _, c := range cmds {
|
||||
if c.Kind == CommandClarify {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func finishRates(m *CategoryMetrics) {
|
||||
m.Top1HitRate = ratio(m.Top1Hits, m.Top1Expected)
|
||||
m.ClarificationRate = ratio(m.Clarifications, m.Turns)
|
||||
m.FallbackRate = ratio(m.Fallbacks, m.Turns)
|
||||
m.GenerationAgreementRate = ratio(m.GenerationAgree, m.GenerationCases)
|
||||
}
|
||||
|
||||
// ratio 定点 4 位舍入,保证重跑 JSON 字节稳定;分母为 0 返回 nil。
|
||||
func ratio(n, d int) *float64 {
|
||||
if d == 0 {
|
||||
return nil
|
||||
}
|
||||
v := math.Round(float64(n)/float64(d)*10000) / 10000
|
||||
return &v
|
||||
}
|
||||
|
||||
// Table 返回人类可读的指标表(每类别一行 + overall 行)。
|
||||
func (s EvalSummary) Table() string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "指标表(corpus %s):\n", s.CorpusDigest)
|
||||
fmt.Fprintf(&b, " %-24s %5s %5s %5s %-14s %8s %-14s %-14s %s\n",
|
||||
"category", "cases", "pass", "turns", "top1", "mis_exec", "clarify", "fallback", "gen_agree")
|
||||
rows := append(append([]CategoryMetrics(nil), s.Categories...), s.Overall)
|
||||
for _, m := range rows {
|
||||
fmt.Fprintf(&b, " %-24s %5d %5d %5d %-14s %8d %-14s %-14s %s\n",
|
||||
m.Category, m.Cases, m.PassedCases, m.Turns,
|
||||
rateCell(m.Top1Hits, m.Top1Expected, m.Top1HitRate),
|
||||
m.MisExecutions,
|
||||
rateCell(m.Clarifications, m.Turns, m.ClarificationRate),
|
||||
rateCell(m.Fallbacks, m.Turns, m.FallbackRate),
|
||||
rateCell(m.GenerationAgree, m.GenerationCases, m.GenerationAgreementRate))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func rateCell(n, d int, r *float64) string {
|
||||
if r == nil {
|
||||
return "-"
|
||||
}
|
||||
return fmt.Sprintf("%d/%d=%.3f", n, d, *r)
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
package intent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func rejectReason(tr Trace, id string) string {
|
||||
for _, r := range tr.RulesRejected {
|
||||
if r.ID == id {
|
||||
return r.Reason
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func buildSingleSpecSnap(t *testing.T, opts Options, specs ...IntentSpec) *Snapshot {
|
||||
t.Helper()
|
||||
b := NewRegistryBuilder(opts)
|
||||
if err := b.Register(specs...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reg, err := b.Build()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return reg.Snapshot()
|
||||
}
|
||||
|
||||
func TestNegatedHitsRejectSpec(t *testing.T) {
|
||||
_, snap := buildDemoRegistry(t)
|
||||
rec := NewRecognizer(Options{}, nil, nil)
|
||||
cases := []struct{ input, id string }{
|
||||
{"不要重启任务", "demo.run.restart"},
|
||||
{"别取消任务 42", "demo.run.cancel"},
|
||||
{"不用查进度了", "demo.run.status_query"},
|
||||
{"先不要发布", "demo.misc.deploy"},
|
||||
{"刚才那个任务先别重启", "demo.run.restart"},
|
||||
{"无需取消任务 42", "demo.run.cancel"},
|
||||
{"不需要查看状态", "demo.run.status_query"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := rejectReason(snap.Explain(tc.input), tc.id); got != "negated_match" {
|
||||
t.Errorf("%q: %s reject = %q, want negated_match", tc.input, tc.id, got)
|
||||
}
|
||||
res, err := rec.Recognize(context.Background(), tc.input, snap)
|
||||
if err != nil || res.Branch != BranchFallback {
|
||||
t.Errorf("%q: branch = %s top = %+v err = %v, want fallback", tc.input, res.Branch, res.Top, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNegationCountsUnnegatedOccurrence(t *testing.T) {
|
||||
_, snap := buildDemoRegistry(t)
|
||||
rec := NewRecognizer(Options{}, nil, nil)
|
||||
cases := []struct{ input, id string }{
|
||||
// 同一词条一次被否定、一次未被否定:未被否定的那次照常命中。
|
||||
{"不要重启任务,还是重启任务吧", "demo.run.restart"},
|
||||
// 否定词指向别处,与命中相隔超过窗口。
|
||||
{"重启任务,不要密码", "demo.run.restart"},
|
||||
{"不要密码,重启任务", "demo.run.restart"},
|
||||
// 「别」在复合词里不表否定。
|
||||
{"识别任务进度", "demo.run.status_query"},
|
||||
{"别的进度", "demo.run.status_query"},
|
||||
// 非否定输入行为不变。
|
||||
{"把任务重启一下", "demo.run.restart"},
|
||||
{"取消任务 42", "demo.run.cancel"},
|
||||
{"查询任务 42 的进度", "demo.run.status_query"},
|
||||
{"please DEPLOY it", "demo.misc.deploy"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
res, err := rec.Recognize(context.Background(), tc.input, snap)
|
||||
if err != nil || res.Branch != BranchMatched || res.Top == nil || res.Top.ID != tc.id {
|
||||
t.Errorf("%q: branch = %s top = %+v err = %v, want matched %s", tc.input, res.Branch, res.Top, err, tc.id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNegatedPatternHit(t *testing.T) {
|
||||
snap := buildSingleSpecSnap(t, Options{}, IntentSpec{
|
||||
ID: "demo.run.restart_re", Domain: "demo", Version: 1,
|
||||
Matcher: MatchSpec{Patterns: []string{`重启\s*任务`}},
|
||||
})
|
||||
if got := rejectReason(snap.Explain("别重启 任务"), "demo.run.restart_re"); got != "negated_match" {
|
||||
t.Fatalf("negated pattern reject = %q", got)
|
||||
}
|
||||
tr := snap.Explain("先别重启任务,算了还是重启任务")
|
||||
if tr.Branch != BranchMatched || tr.TopID != "demo.run.restart_re" {
|
||||
t.Fatalf("un-negated pattern occurrence must match: %+v", tr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGuardIgnoresNegation(t *testing.T) {
|
||||
snap := buildSingleSpecSnap(t, Options{GuardDomains: []string{"guard"}}, IntentSpec{
|
||||
ID: "guard.config_change", Domain: "guard", Version: 1,
|
||||
Matcher: MatchSpec{IncludesAny: []string{"改密码"}},
|
||||
})
|
||||
res, err := NewRecognizer(Options{}, nil, nil).Recognize(context.Background(), "不要改密码", snap)
|
||||
if err != nil || res.Branch != BranchGuardHit {
|
||||
t.Fatalf("guard must still hit on negated phrase: %+v %v", res, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTieBreakPrefersExtractedSlots(t *testing.T) {
|
||||
_, snap := buildDemoRegistry(t)
|
||||
rec := NewRecognizer(Options{}, nil, nil)
|
||||
|
||||
// 寒暄(priority 10、无槽位)与报告同分:抽到主题的报告胜出。
|
||||
in := "你好,帮我生成报告,主题是电机"
|
||||
res, err := rec.Recognize(context.Background(), in, snap)
|
||||
if err != nil || res.Branch != BranchMatched || res.Top == nil || res.Top.ID != "report.generate" {
|
||||
t.Fatalf("greet+report: branch = %s top = %+v err = %v", res.Branch, res.Top, err)
|
||||
}
|
||||
if !reflect.DeepEqual(res.Top.Slots, map[string]string{"topic": "电机"}) {
|
||||
t.Fatalf("slots = %+v", res.Top.Slots)
|
||||
}
|
||||
if tr := snap.Explain(in); tr.Branch != BranchMatched || tr.TopID != "report.generate" {
|
||||
t.Fatalf("explain = %+v", tr)
|
||||
}
|
||||
|
||||
// 同 priority 的两个动作即使槽位数不同也仍澄清;有更高 priority 的寒暄在场也不改变。
|
||||
for _, in := range []string{"取消任务 42 并发布", "你好,取消任务 42 并发布"} {
|
||||
res, err = rec.Recognize(context.Background(), in, snap)
|
||||
if err != nil || res.Branch != BranchAmbiguous {
|
||||
t.Fatalf("%q: branch = %s top = %+v, want ambiguous", in, res.Branch, res.Top)
|
||||
}
|
||||
}
|
||||
|
||||
// 槽位数相同时退回 priority(I5 不变)。
|
||||
res, err = rec.Recognize(context.Background(), "你好呀", snap)
|
||||
if err != nil || res.Top == nil || res.Top.ID != "demo.greet.hello" {
|
||||
t.Fatalf("greet priority: %+v %v", res, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlotInvalidCapturesClarify(t *testing.T) {
|
||||
_, snap := buildDemoRegistry(t)
|
||||
rec := NewRecognizer(Options{}, nil, nil)
|
||||
cases := []struct{ input, top, key, reason string }{
|
||||
{"取消任务 42 和任务 43", "demo.run.cancel", "run_id", "multiple candidate values"},
|
||||
{"取消任务 42 和 43", "demo.run.cancel", "run_id", "multiple candidate values"},
|
||||
{"查询任务 42 还是任务 43 的进度", "demo.run.status_query", "run_id", "multiple candidate values"},
|
||||
{"查询任务 abc_def 的进度", "demo.run.status_query", "run_id", "value truncated by slot charset"},
|
||||
{"取消任务 --", "demo.run.cancel", "run_id", "value has no letters or digits"},
|
||||
{"生成报告,主题是电机和水泵", "report.generate", "topic", "multiple candidate values"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
res, err := rec.Recognize(context.Background(), tc.input, snap)
|
||||
if err != nil || res.Branch != BranchMatched || res.Top == nil || res.Top.ID != tc.top {
|
||||
t.Fatalf("%q: branch = %s top = %+v err = %v", tc.input, res.Branch, res.Top, err)
|
||||
}
|
||||
if got := res.Top.InvalidSlots[tc.key]; got != tc.reason {
|
||||
t.Errorf("%q: invalid[%s] = %q, want %q (slots %+v)", tc.input, tc.key, got, tc.reason, res.Top.Slots)
|
||||
}
|
||||
if _, kept := res.Top.Slots[tc.key]; kept {
|
||||
t.Errorf("%q: invalid value must not be kept: %+v", tc.input, res.Top.Slots)
|
||||
}
|
||||
if len(res.Commands) != 2 || res.Commands[1].Kind != CommandClarify || !reflect.DeepEqual(res.Commands[1].Missing, []string{tc.key}) {
|
||||
t.Errorf("%q: commands = %+v, want start + clarify(%s)", tc.input, res.Commands, tc.key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlotValidCapturesUnchanged(t *testing.T) {
|
||||
_, snap := buildDemoRegistry(t)
|
||||
rec := NewRecognizer(Options{}, nil, nil)
|
||||
cases := []struct {
|
||||
input string
|
||||
slots map[string]string
|
||||
}{
|
||||
{"取消任务 42", map[string]string{"run_id": "42"}},
|
||||
{"查询任务 42 的进度", map[string]string{"run_id": "42"}},
|
||||
{"查询任务42的进度", map[string]string{"run_id": "42"}},
|
||||
{"查询任务 run-7 的进度", map[string]string{"run_id": "run-7"}},
|
||||
{"生成报告,主题是和平", map[string]string{"topic": "和平"}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
res, err := rec.Recognize(context.Background(), tc.input, snap)
|
||||
if err != nil || res.Top == nil || !reflect.DeepEqual(res.Top.Slots, tc.slots) || len(res.Top.InvalidSlots) != 0 {
|
||||
t.Errorf("%q: top = %+v err = %v, want slots %v", tc.input, res.Top, err, tc.slots)
|
||||
}
|
||||
}
|
||||
// 缺失与中文数字仍进 missing。
|
||||
for _, in := range []string{"取消任务", "取消任务四十二"} {
|
||||
res, err := rec.Recognize(context.Background(), in, snap)
|
||||
if err != nil || res.Top == nil || !reflect.DeepEqual(res.Top.MissingRequiredSlots, []string{"run_id"}) {
|
||||
t.Errorf("%q: top = %+v err = %v, want missing run_id", in, res.Top, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlotBlankAfterNormalizeIsMissing(t *testing.T) {
|
||||
snap := buildSingleSpecSnap(t, Options{}, IntentSpec{
|
||||
ID: "report.generate", Domain: "report", Version: 1,
|
||||
Matcher: MatchSpec{IncludesAny: []string{"生成报告"}},
|
||||
Slots: []SlotSpec{{Key: "topic", Required: true, Pattern: `主题是(.+)`, Normalize: []string{"trim"}}},
|
||||
})
|
||||
res, err := NewRecognizer(Options{}, nil, nil).Recognize(context.Background(), "生成报告,主题是 ", snap)
|
||||
if err != nil || res.Top == nil || !reflect.DeepEqual(res.Top.MissingRequiredSlots, []string{"topic"}) || len(res.Top.InvalidSlots) != 0 {
|
||||
t.Fatalf("blank topic must be missing: %+v %v", res.Top, err)
|
||||
}
|
||||
}
|
||||
@@ -132,7 +132,9 @@ func TestR2OverlappingSlotsAreAmbiguous(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestR2NegationIsOnlyExcludesAny(t *testing.T) {
|
||||
// 否定修复(negationTokens)之前,没有 excludes_any 的 spec 会把「不要写入文件」当肯定命中;
|
||||
// 现在紧邻否定词的那次命中不计正向,excludes_any 仍按原语义整体排除。
|
||||
func TestR2NegationTokensAndExcludesAny(t *testing.T) {
|
||||
plain := IntentSpec{
|
||||
ID: "demo.write.file", Domain: "demo", Version: 1,
|
||||
Matcher: MatchSpec{IncludesAny: []string{"写入"}},
|
||||
@@ -150,9 +152,13 @@ func TestR2NegationIsOnlyExcludesAny(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hit, err := rec.Recognize(context.Background(), "不要写入文件", built.Snapshot())
|
||||
negated, err := rec.Recognize(context.Background(), "不要写入文件", built.Snapshot())
|
||||
if err != nil || negated.Branch != BranchFallback {
|
||||
t.Fatalf("negated phrase must not match even without excludes_any: %+v %v", negated, err)
|
||||
}
|
||||
hit, err := rec.Recognize(context.Background(), "写入文件", built.Snapshot())
|
||||
if err != nil || hit.Top == nil || hit.Top.ID != plain.ID {
|
||||
t.Fatalf("without excludes_any the negated phrase still matches: %+v %v", hit, err)
|
||||
t.Fatalf("un-negated phrase must still match: %+v %v", hit, err)
|
||||
}
|
||||
|
||||
exBuilder := NewRegistryBuilder(Options{})
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
package intent
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// recallCorpusDoc 与 intentctl eval 读取的语料结构一致(自声明守卫 spec 与域)。
|
||||
type recallCorpusDoc struct {
|
||||
RecordOnly bool `json:"record_only"`
|
||||
Cases []CorpusCase `json:"cases"`
|
||||
BuiltinSpecs []IntentSpec `json:"builtin_specs"`
|
||||
GuardDomains []string `json:"guard_domains"`
|
||||
ProtectedDomains []string `json:"protected_domains"`
|
||||
}
|
||||
|
||||
var recallCategories = []string{
|
||||
"expired_confirmation",
|
||||
"generation_consistency",
|
||||
"multi_intent",
|
||||
"negation_reference",
|
||||
"wrong_slot",
|
||||
}
|
||||
|
||||
const (
|
||||
recallCorpusPath = "testdata/recall_corpus.json"
|
||||
recallBaselinePath = "../evaluation/metrics.json"
|
||||
)
|
||||
|
||||
// runRecallCorpus 按 intentctl eval 同款方式建快照(语料 builtin_specs + 夹具 specs)
|
||||
// 并逐例执行,保证测试与 CLI 的指标同源。
|
||||
func runRecallCorpus(t *testing.T) ([]byte, recallCorpusDoc, []CaseResult) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(recallCorpusPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var doc recallCorpusDoc
|
||||
if err := json.Unmarshal(data, &doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := NewRegistryBuilder(Options{GuardDomains: doc.GuardDomains, ProtectedDomains: doc.ProtectedDomains})
|
||||
if err := b.Register(doc.BuiltinSpecs...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reg, err := b.BuildFS(context.Background(), os.DirFS("testdata"), "specs")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
snap := reg.Snapshot()
|
||||
results := make([]CaseResult, 0, len(doc.Cases))
|
||||
for _, tc := range doc.Cases {
|
||||
results = append(results, RunCorpusCase(context.Background(), snap, tc))
|
||||
}
|
||||
return data, doc, results
|
||||
}
|
||||
|
||||
// TestRecallCorpusStructure 只断言结构覆盖(A3):≥30 例、五类齐全、每例可执行、
|
||||
// 指标 JSON 合法且与用例计数一致;不断言任何质量数值(基线只记录)。
|
||||
func TestRecallCorpusStructure(t *testing.T) {
|
||||
data, doc, results := runRecallCorpus(t)
|
||||
if !doc.RecordOnly {
|
||||
t.Fatal("召回基线语料须声明 record_only")
|
||||
}
|
||||
if len(doc.Cases) < 30 {
|
||||
t.Fatalf("cases = %d, want >= 30", len(doc.Cases))
|
||||
}
|
||||
perCat := map[string]int{}
|
||||
for i, tc := range doc.Cases {
|
||||
perCat[tc.Category]++
|
||||
turns := corpusTurns(tc)
|
||||
res := results[i]
|
||||
if len(res.Turns) != len(turns) {
|
||||
t.Fatalf("case %s executed %d/%d turns", tc.Name, len(res.Turns), len(turns))
|
||||
}
|
||||
for _, tr := range res.Turns {
|
||||
for _, mm := range tr.Mismatches {
|
||||
if strings.HasPrefix(mm, "识别错误") {
|
||||
t.Fatalf("case %s turn %d: %s", tc.Name, tr.Index, mm)
|
||||
}
|
||||
}
|
||||
}
|
||||
if tc.Category == "generation_consistency" {
|
||||
if len(tc.GeneratorCommands) == 0 || len(turns) != 1 || res.Generator == nil {
|
||||
t.Fatalf("case %s: generation_consistency 须为带 generator_commands 的单轮用例", tc.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, cat := range recallCategories {
|
||||
if perCat[cat] == 0 {
|
||||
t.Fatalf("category %s missing (have %v)", cat, perCat)
|
||||
}
|
||||
}
|
||||
if len(perCat) != len(recallCategories) {
|
||||
t.Fatalf("unexpected categories %v, want exactly %v", perCat, recallCategories)
|
||||
}
|
||||
|
||||
summary := SummarizeCorpus(CorpusDigest(data), doc.Cases, results)
|
||||
t.Log("\n" + summary.Table())
|
||||
raw, err := json.Marshal(summary)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checkSummaryJSON(t, "fresh", raw, CorpusDigest(data), doc.Cases)
|
||||
|
||||
baseline, err := os.ReadFile(recallBaselinePath)
|
||||
if err != nil {
|
||||
t.Fatalf("read committed baseline: %v", err)
|
||||
}
|
||||
checkSummaryJSON(t, "baseline", baseline, CorpusDigest(data), doc.Cases)
|
||||
}
|
||||
|
||||
// checkSummaryJSON 校验指标 JSON 结构与计数一致性(不比较质量数值)。
|
||||
// 已提交基线的摘要必须等于当前语料摘要:改语料后须按 evaluation/BASELINE.md 重跑。
|
||||
func checkSummaryJSON(t *testing.T, label string, raw []byte, digest string, cases []CorpusCase) {
|
||||
t.Helper()
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
dec.DisallowUnknownFields()
|
||||
var s EvalSummary
|
||||
if err := dec.Decode(&s); err != nil {
|
||||
t.Fatalf("%s: metrics JSON malformed: %v", label, err)
|
||||
}
|
||||
if s.Schema != EvalMetricsSchema {
|
||||
t.Fatalf("%s: schema = %q", label, s.Schema)
|
||||
}
|
||||
if s.CorpusDigest != digest {
|
||||
t.Fatalf("%s: corpus_digest = %s, want %s(语料已变更,需重跑基线)", label, s.CorpusDigest, digest)
|
||||
}
|
||||
wantCases, wantTurns, wantGen := map[string]int{}, map[string]int{}, map[string]int{}
|
||||
totalTurns, totalGen := 0, 0
|
||||
for _, tc := range cases {
|
||||
n := len(corpusTurns(tc))
|
||||
wantCases[tc.Category]++
|
||||
wantTurns[tc.Category] += n
|
||||
totalTurns += n
|
||||
if len(tc.GeneratorCommands) > 0 {
|
||||
wantGen[tc.Category]++
|
||||
totalGen++
|
||||
}
|
||||
}
|
||||
if len(s.Categories) != len(wantCases) {
|
||||
t.Fatalf("%s: categories = %d, want %d", label, len(s.Categories), len(wantCases))
|
||||
}
|
||||
sumCases, sumTurns := 0, 0
|
||||
for _, m := range s.Categories {
|
||||
if m.Cases != wantCases[m.Category] || m.Turns != wantTurns[m.Category] || m.GenerationCases != wantGen[m.Category] {
|
||||
t.Fatalf("%s: %s counts = %+v, want cases=%d turns=%d gen=%d", label, m.Category, m, wantCases[m.Category], wantTurns[m.Category], wantGen[m.Category])
|
||||
}
|
||||
checkMetricsBounds(t, label, m)
|
||||
sumCases += m.Cases
|
||||
sumTurns += m.Turns
|
||||
}
|
||||
o := s.Overall
|
||||
if o.Cases != len(cases) || sumCases != o.Cases || o.Turns != totalTurns || sumTurns != o.Turns || o.GenerationCases != totalGen {
|
||||
t.Fatalf("%s: overall = %+v, want cases=%d turns=%d gen=%d", label, o, len(cases), totalTurns, totalGen)
|
||||
}
|
||||
checkMetricsBounds(t, label, o)
|
||||
}
|
||||
|
||||
func checkMetricsBounds(t *testing.T, label string, m CategoryMetrics) {
|
||||
t.Helper()
|
||||
pairs := []struct {
|
||||
name string
|
||||
n, d int
|
||||
rate *float64
|
||||
}{
|
||||
{"top1", m.Top1Hits, m.Top1Expected, m.Top1HitRate},
|
||||
{"clarification", m.Clarifications, m.Turns, m.ClarificationRate},
|
||||
{"fallback", m.Fallbacks, m.Turns, m.FallbackRate},
|
||||
{"generation", m.GenerationAgree, m.GenerationCases, m.GenerationAgreementRate},
|
||||
}
|
||||
for _, p := range pairs {
|
||||
if p.n < 0 || p.n > p.d {
|
||||
t.Fatalf("%s: %s %s = %d/%d out of range", label, m.Category, p.name, p.n, p.d)
|
||||
}
|
||||
want := ratio(p.n, p.d)
|
||||
if (want == nil) != (p.rate == nil) || (want != nil && *want != *p.rate) {
|
||||
t.Fatalf("%s: %s %s rate inconsistent with %d/%d", label, m.Category, p.name, p.n, p.d)
|
||||
}
|
||||
}
|
||||
if m.PassedCases > m.Cases || m.MisExecutions > m.Turns || m.Top1Expected > m.Turns {
|
||||
t.Fatalf("%s: %s counts out of range: %+v", label, m.Category, m)
|
||||
}
|
||||
}
|
||||
+39
-9
@@ -72,7 +72,8 @@ func (r *FunnelRecognizer) recognize(ctx context.Context, input string, snap *Sn
|
||||
if strings.TrimSpace(input) == "" {
|
||||
return Result{Input: input, Branch: BranchFallback, SnapshotRevision: snap.rev}
|
||||
}
|
||||
// 否定只认 excludes_any 子串,不做语义否定;「不要写入」这类说法交给命令层 Cancel。
|
||||
// 否定两处生效:excludes_any 整体排除;includes_any/pattern 命中前紧邻否定词
|
||||
// (negationTokens)的那次命中不算正向命中。
|
||||
matchInput, folded := snap.foldInput(input)
|
||||
|
||||
// Stage 0:守卫命中即短路,可表达"识别即拒绝"。
|
||||
@@ -104,7 +105,7 @@ func (r *FunnelRecognizer) recognize(ctx context.Context, input string, snap *Sn
|
||||
}
|
||||
}
|
||||
return Result{Input: input, Branch: BranchFallback, SnapshotRevision: snap.rev}
|
||||
case len(hits) == 1 || hits[0].score-hits[1].score >= snap.margin || tieBreakByPriority(hits):
|
||||
case len(hits) == 1 || hits[0].score-hits[1].score >= snap.margin || resolveTie(hits, input):
|
||||
cands := candidatesOf(trimHits(hits, snap.topN), input, true)
|
||||
top := cands[0]
|
||||
return Result{Input: input, Branch: BranchMatched, Top: &top, Candidates: cands, SnapshotRevision: snap.rev}
|
||||
@@ -141,14 +142,43 @@ func (r *FunnelRecognizer) recognize(ctx context.Context, input string, snap *Sn
|
||||
return Result{Input: input, Branch: BranchAmbiguous, Candidates: cands, SnapshotRevision: snap.rev}
|
||||
}
|
||||
|
||||
// tieBreakByPriority 报告同分(分差为 0)且前两名 Priority 不同:
|
||||
// 此时高 Priority 者直接胜出,不再标歧(I5)。分差小于 margin 但非零时
|
||||
// 仍按 ambiguous 处理(margin 语义不变)。
|
||||
func tieBreakByPriority(hits []specHit) bool {
|
||||
if len(hits) < 2 {
|
||||
// resolveTie 裁决与第一名同分(分差为 0)的候选组(I5 + 特异度)。
|
||||
// 组内按(对 input 成功抽取的槽位数降序、Priority 降序、合并序)排名;排名第一者须
|
||||
// 胜过组内每一个其余候选才直接胜出、不再标歧,此时把它移到 hits 首位并返回 true。
|
||||
// a 胜过 b 当且仅当二者 Priority 不同,且 a 抽到的槽位更多、或槽位数相同而 Priority 更高:
|
||||
// 寒暄(高 Priority、无槽位)让位于抽到槽位的动作;Priority 相同的两个动作仍按
|
||||
// ambiguous 澄清(多意图输入),候选次序不变。分差小于 margin 但非零时仍按
|
||||
// ambiguous 处理(margin 语义不变)。
|
||||
func resolveTie(hits []specHit, input string) bool {
|
||||
if len(hits) < 2 || hits[0].score != hits[1].score {
|
||||
return false
|
||||
}
|
||||
return hits[0].score == hits[1].score && hits[0].cs.spec.Priority > hits[1].cs.spec.Priority
|
||||
n := 2
|
||||
for n < len(hits) && hits[n].score == hits[0].score {
|
||||
n++
|
||||
}
|
||||
filled := make(map[*compiledSpec]int, n)
|
||||
for _, h := range hits[:n] {
|
||||
slots, _, _ := h.cs.extractSlots(input)
|
||||
filled[h.cs] = len(slots)
|
||||
}
|
||||
ranked := append([]specHit(nil), hits[:n]...)
|
||||
sort.SliceStable(ranked, func(i, j int) bool {
|
||||
if a, b := filled[ranked[i].cs], filled[ranked[j].cs]; a != b {
|
||||
return a > b
|
||||
}
|
||||
return lessHit(ranked[i], ranked[j])
|
||||
})
|
||||
best := ranked[0]
|
||||
for _, other := range ranked[1:] {
|
||||
pa, pb := best.cs.spec.Priority, other.cs.spec.Priority
|
||||
fa, fb := filled[best.cs], filled[other.cs]
|
||||
if pa == pb || fa < fb || (fa == fb && pa < pb) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
copy(hits[:n], ranked)
|
||||
return true
|
||||
}
|
||||
|
||||
// validateRefined 要求分类器返回与原候选完全相同的 ID 集合且无重复。
|
||||
@@ -279,7 +309,7 @@ func (s *Snapshot) Explain(input string) Trace {
|
||||
switch {
|
||||
case len(hits) == 0:
|
||||
trace.Branch = BranchFallback
|
||||
case len(hits) == 1 || hits[0].score-hits[1].score >= s.margin || tieBreakByPriority(hits):
|
||||
case len(hits) == 1 || hits[0].score-hits[1].score >= s.margin || resolveTie(hits, input):
|
||||
trace.Branch = BranchMatched
|
||||
trace.TopID = hits[0].cs.spec.ID
|
||||
default:
|
||||
|
||||
+105
-3
@@ -10,6 +10,7 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"unicode/utf8"
|
||||
|
||||
"logging-system/logsys"
|
||||
)
|
||||
@@ -151,20 +152,29 @@ func (cs *compiledSpec) evaluate(input, folded string) matchOutcome {
|
||||
}
|
||||
}
|
||||
var hitTerms, hitPat []string
|
||||
positive := false
|
||||
positive, negated := false, false
|
||||
for _, t := range cs.termsAny {
|
||||
if cs.contains(hay, t) {
|
||||
switch cs.termHit(hay, t) {
|
||||
case hitPositive:
|
||||
positive = true
|
||||
hitTerms = append(hitTerms, t)
|
||||
case hitNegated:
|
||||
negated = true
|
||||
}
|
||||
}
|
||||
for i, p := range cs.patterns {
|
||||
if p.MatchString(input) {
|
||||
switch cs.patternHit(p, input) {
|
||||
case hitPositive:
|
||||
positive = true
|
||||
hitPat = append(hitPat, cs.spec.Matcher.Patterns[i])
|
||||
case hitNegated:
|
||||
negated = true
|
||||
}
|
||||
}
|
||||
if !positive && (len(cs.termsAny) > 0 || len(cs.patterns) > 0) {
|
||||
if negated {
|
||||
return matchOutcome{reject: "negated_match"}
|
||||
}
|
||||
return matchOutcome{reject: "no_positive_match"}
|
||||
}
|
||||
score := cs.base
|
||||
@@ -186,6 +196,98 @@ func (cs *compiledSpec) contains(hay, term string) bool {
|
||||
return strings.Contains(hay, term)
|
||||
}
|
||||
|
||||
// negationTokens 否定词:includes_any 词条或 pattern 的某次命中前紧邻、或相隔至多
|
||||
// negationWindow 个字符处出现其中之一时,该次命中视为被否定,不计入正向命中
|
||||
// (「不要重启任务」「别取消任务 42」「不用查进度了」不触发对应 spec)。
|
||||
// 逐次判断:同一词条只要还有一次未被否定的出现就照常命中。守卫域 spec 不做否定判断
|
||||
// (守卫宁可多拦),excludes_any 与 includes_all 也不受影响。
|
||||
var negationTokens = []string{"不要", "别", "不用", "先不要", "先别", "无需", "不需要"}
|
||||
|
||||
// negationWindow 否定词末尾与命中起点之间允许间隔的最大字符(rune)数。
|
||||
const negationWindow = 2
|
||||
|
||||
// nonNegatingBie 含「别」但不表否定的常见词:单字否定词「别」落在这些词里时不算否定,
|
||||
// 避免「识别任务进度」「别的任务进度」被误拦。
|
||||
var nonNegatingBie = []string{
|
||||
"区别", "特别", "识别", "类别", "级别", "分别", "个别", "告别", "差别", "性别", "辨别", "鉴别",
|
||||
"别的", "别人", "别处", "别名",
|
||||
}
|
||||
|
||||
// hitKind 一个词条或 pattern 对输入的命中情况。
|
||||
type hitKind int
|
||||
|
||||
const (
|
||||
hitNone hitKind = iota
|
||||
hitNegated // 有出现,但每次出现都被否定
|
||||
hitPositive // 至少一次未被否定的出现
|
||||
)
|
||||
|
||||
// termHit 逐次检查 term 在 hay 中的出现(按 boundary 配置判词界),区分正向与被否定命中。
|
||||
func (cs *compiledSpec) termHit(hay, term string) hitKind {
|
||||
if term == "" {
|
||||
if cs.contains(hay, term) {
|
||||
return hitPositive
|
||||
}
|
||||
return hitNone
|
||||
}
|
||||
kind := hitNone
|
||||
for i := strings.Index(hay, term); i >= 0; {
|
||||
if !cs.spec.Matcher.Boundary || boundedAt(hay, i, term) {
|
||||
if cs.guard || !negatedAt(hay, i) {
|
||||
return hitPositive
|
||||
}
|
||||
kind = hitNegated
|
||||
}
|
||||
j := strings.Index(hay[i+1:], term)
|
||||
if j < 0 {
|
||||
break
|
||||
}
|
||||
i += 1 + j
|
||||
}
|
||||
return kind
|
||||
}
|
||||
|
||||
// patternHit 逐个检查 pattern 在 input 中的匹配,按匹配起点判否定。
|
||||
func (cs *compiledSpec) patternHit(p *regexp.Regexp, input string) hitKind {
|
||||
kind := hitNone
|
||||
for _, loc := range p.FindAllStringIndex(input, -1) {
|
||||
if cs.guard || !negatedAt(input, loc[0]) {
|
||||
return hitPositive
|
||||
}
|
||||
kind = hitNegated
|
||||
}
|
||||
return kind
|
||||
}
|
||||
|
||||
// negatedAt 报告 s[pos:] 处的命中前方 negationWindow 个字符内是否以否定词结尾。
|
||||
func negatedAt(s string, pos int) bool {
|
||||
prefix := s[:pos]
|
||||
for gap := 0; gap <= negationWindow; gap++ {
|
||||
for _, tok := range negationTokens {
|
||||
if strings.HasSuffix(prefix, tok) && !(tok == "别" && inNonNegatingBie(s, len(prefix)-len(tok))) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if prefix == "" {
|
||||
return false
|
||||
}
|
||||
_, size := utf8.DecodeLastRuneInString(prefix)
|
||||
prefix = prefix[:len(prefix)-size]
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// inNonNegatingBie 报告 s[at:] 处的「别」是否属于 nonNegatingBie 中的某个词。
|
||||
func inNonNegatingBie(s string, at int) bool {
|
||||
for _, w := range nonNegatingBie {
|
||||
k := strings.Index(w, "别")
|
||||
if at-k >= 0 && strings.HasPrefix(s[at-k:], w) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func compileSpec(spec IntentSpec, guard bool, index int, scoped map[string]SlotValidator, nfkc bool) (*compiledSpec, error) {
|
||||
m := spec.Matcher
|
||||
fold := func(terms []string) []string {
|
||||
|
||||
+115
-2
@@ -4,6 +4,8 @@ import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// SlotValidator 宿主注册的槽位值校验器。必须纯函数、确定性、无 I/O。
|
||||
@@ -101,7 +103,8 @@ type slotCapture struct {
|
||||
}
|
||||
|
||||
// extractSlots 抽取并归一化槽位。
|
||||
// missing:必填槽位未捕获(模式没匹配上);invalid:捕获到但校验不合法,或与另一槽位区间重叠(key→原因)。
|
||||
// missing:必填槽位未捕获(模式没匹配上,或归一化后为空);invalid:捕获到但不可用(key→原因)——
|
||||
// 校验不合法、与另一槽位区间重叠、含多个候选值、只有符号、或被槽位字符集截断(见 captureIssue)。
|
||||
// 必填且非法只进 invalid(宿主可区分"没给"与"给错了"来决定澄清话术)。
|
||||
func (cs *compiledSpec) extractSlots(input string) (slots map[string]string, missing []string, invalid map[string]string) {
|
||||
if len(cs.slots) == 0 {
|
||||
@@ -117,7 +120,14 @@ func (cs *compiledSpec) extractSlots(input string) (slots map[string]string, mis
|
||||
}
|
||||
raw := input[loc[2*g]:loc[2*g+1]]
|
||||
v := normalizeSlotValue(raw, s.normalize)
|
||||
reason, ok := s.validate(v)
|
||||
if strings.TrimSpace(v) == "" {
|
||||
caps = append(caps, slotCapture{key: s.key, required: s.required})
|
||||
continue
|
||||
}
|
||||
reason, ok := s.captureIssue(input, loc, v)
|
||||
if ok {
|
||||
reason, ok = s.validate(v)
|
||||
}
|
||||
caps = append(caps, slotCapture{
|
||||
key: s.key, start: loc[2*g], end: loc[2*g+1], value: v,
|
||||
ok: ok, reason: reason, required: s.required, matched: true,
|
||||
@@ -167,6 +177,109 @@ func (cs *compiledSpec) extractSlots(input string) (slots map[string]string, mis
|
||||
return slots, missing, invalid
|
||||
}
|
||||
|
||||
// slotValueSeparators 多值分隔词:出现在槽位值内部(两侧都有内容),或紧跟在捕获值之后
|
||||
// 且其后又能抽出同一槽位时,说明用户给了多个候选值(「任务 42 还是任务 43」)。
|
||||
var slotValueSeparators = []string{"还是", "和", "、", ",", ","}
|
||||
|
||||
// captureIssue 检查一次捕获是否可用(在 format/enum/validator 之前);返回(原因, 是否可用)。
|
||||
// loc 为槽位正则在 input 上的 FindStringSubmatchIndex 结果,v 为归一化后的值。
|
||||
func (s *compiledSlot) captureIssue(input string, loc []int, v string) (string, bool) {
|
||||
if captureTruncated(input, loc, s.group) {
|
||||
return "value truncated by slot charset", false
|
||||
}
|
||||
if hasInnerSeparator(v) || s.repeatsAfter(input, loc) {
|
||||
return "multiple candidate values", false
|
||||
}
|
||||
if !hasValueRune(v) {
|
||||
return "value has no letters or digits", false
|
||||
}
|
||||
return "", true
|
||||
}
|
||||
|
||||
// captureTruncated 报告捕获组是否在一个 ASCII 词中间被槽位字符集截断(`abc_def` 只抽到 `abc`)。
|
||||
// 只在捕获组边界即整个匹配边界时检查:模式自身在组外继续消费的字符不算截断。
|
||||
func captureTruncated(input string, loc []int, g int) bool {
|
||||
start, end := loc[2*g], loc[2*g+1]
|
||||
if end == loc[1] && end < len(input) {
|
||||
if r, _ := utf8.DecodeRuneInString(input[end:]); isASCIIWord(r) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if start == loc[0] && start > 0 {
|
||||
if r, _ := utf8.DecodeLastRuneInString(input[:start]); isASCIIWord(r) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isASCIIWord(r rune) bool {
|
||||
return r == '_' || (r < utf8.RuneSelf && (unicode.IsLetter(r) || unicode.IsDigit(r)))
|
||||
}
|
||||
|
||||
// hasInnerSeparator 报告 v 内部是否有两侧都非空的多值分隔词(「电机和水泵」)。
|
||||
func hasInnerSeparator(v string) bool {
|
||||
for _, sep := range slotValueSeparators {
|
||||
for i := strings.Index(v, sep); i >= 0; {
|
||||
if strings.TrimSpace(v[:i]) != "" && strings.TrimSpace(v[i+len(sep):]) != "" {
|
||||
return true
|
||||
}
|
||||
j := strings.Index(v[i+len(sep):], sep)
|
||||
if j < 0 {
|
||||
break
|
||||
}
|
||||
i += len(sep) + j
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// repeatsAfter 报告捕获值之后是否紧跟同一槽位的另一个候选值:
|
||||
// - 可选分隔词后完整重复槽位模式(「任务 42 还是任务 43」「任务 42 任务 43」);
|
||||
// 无分隔词时要求模式在捕获组前有前缀,避免把「([a-z]+)」后的普通单词当成候选;
|
||||
// - 有分隔词时也认省略前缀的写法(「任务 42 和 43」)。
|
||||
func (s *compiledSlot) repeatsAfter(input string, loc []int) bool {
|
||||
g := s.group
|
||||
rest := strings.TrimLeftFunc(input[loc[2*g+1]:], unicode.IsSpace)
|
||||
sep := false
|
||||
for _, t := range slotValueSeparators {
|
||||
if strings.HasPrefix(rest, t) {
|
||||
rest, sep = strings.TrimLeftFunc(rest[len(t):], unicode.IsSpace), true
|
||||
break
|
||||
}
|
||||
}
|
||||
if rest == "" {
|
||||
return false
|
||||
}
|
||||
prefix := input[loc[0]:loc[2*g]]
|
||||
if sep || prefix != "" {
|
||||
if m := s.re.FindStringSubmatchIndex(rest); m != nil && m[0] == 0 && groupCaptured(m, g) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if sep && prefix != "" {
|
||||
probe := prefix + rest
|
||||
if m := s.re.FindStringSubmatchIndex(probe); m != nil && m[0] == 0 && groupCaptured(m, g) && m[2*g] == len(prefix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func groupCaptured(loc []int, g int) bool {
|
||||
return 2*g+1 < len(loc) && loc[2*g] >= 0 && loc[2*g] < loc[2*g+1]
|
||||
}
|
||||
|
||||
// hasValueRune 报告 v 是否含字母、数字或汉字(只有符号/标点的值不可用)。
|
||||
func hasValueRune(v string) bool {
|
||||
for _, r := range v {
|
||||
if unicode.IsLetter(r) || unicode.IsNumber(r) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// validateProvidedSlots 校验调用方直接给出的槽位表(生成器路径)。
|
||||
// 与 extractSlots 同款结果形态:归一化 → format/enum/validator;必填缺失进 missing;
|
||||
// 捕获到但不合法进 invalid;未知键进 invalid(reason=unknown slot key)。
|
||||
|
||||
Vendored
+575
@@ -0,0 +1,575 @@
|
||||
{
|
||||
"record_only": true,
|
||||
"cases": [
|
||||
{
|
||||
"name": "neg_do_not_restart",
|
||||
"category": "negation_reference",
|
||||
"input": "不要重启任务",
|
||||
"expect_branch": "fallback"
|
||||
},
|
||||
{
|
||||
"name": "neg_do_not_cancel_run",
|
||||
"category": "negation_reference",
|
||||
"input": "别取消任务 42",
|
||||
"expect_branch": "fallback"
|
||||
},
|
||||
{
|
||||
"name": "neg_no_need_progress",
|
||||
"category": "negation_reference",
|
||||
"input": "不用查进度了",
|
||||
"expect_branch": "fallback"
|
||||
},
|
||||
{
|
||||
"name": "neg_do_not_deploy",
|
||||
"category": "negation_reference",
|
||||
"input": "先不要发布",
|
||||
"expect_branch": "fallback"
|
||||
},
|
||||
{
|
||||
"name": "neg_excludes_config_progress",
|
||||
"category": "negation_reference",
|
||||
"input": "进度配置在哪里看",
|
||||
"expect_branch": "fallback"
|
||||
},
|
||||
{
|
||||
"name": "ref_same_as_before_no_context",
|
||||
"category": "negation_reference",
|
||||
"input": "还是刚才那个",
|
||||
"expect_branch": "fallback"
|
||||
},
|
||||
{
|
||||
"name": "ref_same_after_status",
|
||||
"category": "negation_reference",
|
||||
"turns": [
|
||||
{
|
||||
"input": "查询任务 42 的进度",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.run.status_query",
|
||||
"expect_slots": {
|
||||
"run_id": "42"
|
||||
}
|
||||
},
|
||||
{
|
||||
"input": "还是刚才那个,再看下进度",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.run.status_query"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "ref_previous_task_hold_restart",
|
||||
"category": "negation_reference",
|
||||
"turns": [
|
||||
{
|
||||
"input": "查询任务 42 的进度",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.run.status_query"
|
||||
},
|
||||
{
|
||||
"input": "刚才那个任务先别重启",
|
||||
"expect_branch": "fallback"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "multi_status_then_restart",
|
||||
"category": "multi_intent",
|
||||
"input": "查询任务 42 的进度,然后重启任务",
|
||||
"expect_branch": "ambiguous"
|
||||
},
|
||||
{
|
||||
"name": "multi_greet_then_report",
|
||||
"category": "multi_intent",
|
||||
"input": "你好,帮我生成报告,主题是电机",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "report.generate",
|
||||
"expect_slots": {
|
||||
"topic": "电机"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "multi_cancel_and_deploy",
|
||||
"category": "multi_intent",
|
||||
"input": "取消任务 42 并发布",
|
||||
"expect_branch": "ambiguous"
|
||||
},
|
||||
{
|
||||
"name": "multi_greet_then_status",
|
||||
"category": "multi_intent",
|
||||
"input": "你好,查一下任务 7 的进度",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.run.status_query",
|
||||
"expect_slots": {
|
||||
"run_id": "7"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "multi_restart_and_deploy",
|
||||
"category": "multi_intent",
|
||||
"input": "重启任务后再发布",
|
||||
"expect_branch": "ambiguous"
|
||||
},
|
||||
{
|
||||
"name": "multi_status_and_guard",
|
||||
"category": "multi_intent",
|
||||
"input": "查一下进度,顺便改密码",
|
||||
"expect_branch": "guard_hit",
|
||||
"expect_top": "guard.config_change"
|
||||
},
|
||||
{
|
||||
"name": "multi_report_and_cancel",
|
||||
"category": "multi_intent",
|
||||
"input": "生成报告,主题是电机,再取消任务 9",
|
||||
"expect_branch": "ambiguous"
|
||||
},
|
||||
{
|
||||
"name": "slot_cancel_no_id",
|
||||
"category": "wrong_slot",
|
||||
"turns": [
|
||||
{
|
||||
"input": "取消任务",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.run.cancel",
|
||||
"expect_missing_slots": [
|
||||
"run_id"
|
||||
],
|
||||
"expect_commands": [
|
||||
{
|
||||
"kind": "start"
|
||||
},
|
||||
{
|
||||
"kind": "clarify",
|
||||
"missing": [
|
||||
"run_id"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "slot_cancel_chinese_numeral",
|
||||
"category": "wrong_slot",
|
||||
"turns": [
|
||||
{
|
||||
"input": "取消任务四十二",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.run.cancel",
|
||||
"expect_missing_slots": [
|
||||
"run_id"
|
||||
],
|
||||
"expect_commands": [
|
||||
{
|
||||
"kind": "start"
|
||||
},
|
||||
{
|
||||
"kind": "clarify",
|
||||
"missing": [
|
||||
"run_id"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "slot_cancel_two_ids",
|
||||
"category": "wrong_slot",
|
||||
"turns": [
|
||||
{
|
||||
"input": "取消任务 42 和任务 43",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.run.cancel",
|
||||
"expect_commands": [
|
||||
{
|
||||
"kind": "start"
|
||||
},
|
||||
{
|
||||
"kind": "clarify"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "slot_status_two_ids",
|
||||
"category": "wrong_slot",
|
||||
"turns": [
|
||||
{
|
||||
"input": "查询任务 42 还是任务 43 的进度",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.run.status_query",
|
||||
"expect_commands": [
|
||||
{
|
||||
"kind": "start"
|
||||
},
|
||||
{
|
||||
"kind": "clarify"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "slot_status_id_underscore",
|
||||
"category": "wrong_slot",
|
||||
"turns": [
|
||||
{
|
||||
"input": "查询任务 abc_def 的进度",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.run.status_query",
|
||||
"expect_commands": [
|
||||
{
|
||||
"kind": "start"
|
||||
},
|
||||
{
|
||||
"kind": "clarify"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "slot_cancel_dash_only",
|
||||
"category": "wrong_slot",
|
||||
"turns": [
|
||||
{
|
||||
"input": "取消任务 --",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.run.cancel",
|
||||
"expect_commands": [
|
||||
{
|
||||
"kind": "start"
|
||||
},
|
||||
{
|
||||
"kind": "clarify"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "slot_report_empty_topic",
|
||||
"category": "wrong_slot",
|
||||
"turns": [
|
||||
{
|
||||
"input": "生成报告,主题是",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "report.generate",
|
||||
"expect_missing_slots": [
|
||||
"topic"
|
||||
],
|
||||
"expect_commands": [
|
||||
{
|
||||
"kind": "start"
|
||||
},
|
||||
{
|
||||
"kind": "clarify",
|
||||
"missing": [
|
||||
"topic"
|
||||
],
|
||||
"question": "要查询哪个主题?"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "exp_cancel_live_control",
|
||||
"category": "expired_confirmation",
|
||||
"turns": [
|
||||
{
|
||||
"input": "取消一下",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.run.cancel",
|
||||
"expect_missing_slots": [
|
||||
"run_id"
|
||||
]
|
||||
},
|
||||
{
|
||||
"input": "取消任务 42",
|
||||
"expect_branch": "expectation_hit",
|
||||
"expect_top": "demo.run.cancel",
|
||||
"expect_slots": {
|
||||
"run_id": "42"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "exp_cancel_expired_full_command",
|
||||
"category": "expired_confirmation",
|
||||
"turns": [
|
||||
{
|
||||
"input": "取消一下",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.run.cancel"
|
||||
},
|
||||
{
|
||||
"input": "取消任务 42",
|
||||
"expectation_expired": true,
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.run.cancel",
|
||||
"expect_slots": {
|
||||
"run_id": "42"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "exp_cancel_expired_bare_answer",
|
||||
"category": "expired_confirmation",
|
||||
"turns": [
|
||||
{
|
||||
"input": "取消一下",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.run.cancel"
|
||||
},
|
||||
{
|
||||
"input": "就是任务 42",
|
||||
"expectation_expired": true,
|
||||
"expect_branch": "fallback"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "exp_report_expired_bare_topic",
|
||||
"category": "expired_confirmation",
|
||||
"turns": [
|
||||
{
|
||||
"input": "帮我生成报告",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "report.generate"
|
||||
},
|
||||
{
|
||||
"input": "主题是电机",
|
||||
"expectation_expired": true,
|
||||
"expect_branch": "fallback"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "exp_report_expired_full_command",
|
||||
"category": "expired_confirmation",
|
||||
"turns": [
|
||||
{
|
||||
"input": "帮我生成报告",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "report.generate"
|
||||
},
|
||||
{
|
||||
"input": "生成报告,主题是水泵",
|
||||
"expectation_expired": true,
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "report.generate",
|
||||
"expect_slots": {
|
||||
"topic": "水泵"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "exp_restart_confirm_expired",
|
||||
"category": "expired_confirmation",
|
||||
"turns": [
|
||||
{
|
||||
"input": "把任务重启一下",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.run.restart"
|
||||
},
|
||||
{
|
||||
"input": "好的,确认",
|
||||
"pending_expectation": [
|
||||
"demo.run.restart"
|
||||
],
|
||||
"expectation_expired": true,
|
||||
"expect_branch": "fallback"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "exp_restart_confirm_expired_restated",
|
||||
"category": "expired_confirmation",
|
||||
"turns": [
|
||||
{
|
||||
"input": "把任务重启一下",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.run.restart"
|
||||
},
|
||||
{
|
||||
"input": "确认重启任务",
|
||||
"pending_expectation": [
|
||||
"demo.run.restart"
|
||||
],
|
||||
"expectation_expired": true,
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.run.restart"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "gen_status_agree",
|
||||
"category": "generation_consistency",
|
||||
"input": "查询任务 42 的进度",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.run.status_query",
|
||||
"expect_slots": {
|
||||
"run_id": "42"
|
||||
},
|
||||
"generator_commands": [
|
||||
{
|
||||
"kind": "start",
|
||||
"action": {
|
||||
"kind": "handler",
|
||||
"id": "taskrun.query_status"
|
||||
},
|
||||
"slots": {
|
||||
"run_id": "42"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "gen_restart_agree",
|
||||
"category": "generation_consistency",
|
||||
"input": "把任务重启一下",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.run.restart",
|
||||
"generator_commands": [
|
||||
{
|
||||
"kind": "start",
|
||||
"action": {
|
||||
"kind": "auto",
|
||||
"id": "demo.run.restart"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "gen_report_agree",
|
||||
"category": "generation_consistency",
|
||||
"input": "生成报告,主题是电机",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "report.generate",
|
||||
"expect_slots": {
|
||||
"topic": "电机"
|
||||
},
|
||||
"generator_commands": [
|
||||
{
|
||||
"kind": "start",
|
||||
"action": {
|
||||
"kind": "handler",
|
||||
"id": "reportHandler"
|
||||
},
|
||||
"slots": {
|
||||
"topic": "电机"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "gen_deploy_agree",
|
||||
"category": "generation_consistency",
|
||||
"input": "please DEPLOY it",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.misc.deploy",
|
||||
"generator_commands": [
|
||||
{
|
||||
"kind": "start",
|
||||
"action": {
|
||||
"kind": "handler",
|
||||
"id": "deploy.run"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "gen_cancel_slot_disagree",
|
||||
"category": "generation_consistency",
|
||||
"input": "取消任务 42",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.run.cancel",
|
||||
"expect_slots": {
|
||||
"run_id": "42"
|
||||
},
|
||||
"generator_commands": [
|
||||
{
|
||||
"kind": "start",
|
||||
"action": {
|
||||
"kind": "auto",
|
||||
"id": "demo.run.cancel"
|
||||
},
|
||||
"slots": {
|
||||
"run_id": "43"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "gen_progress_wrong_action",
|
||||
"category": "generation_consistency",
|
||||
"input": "帮我看下进度",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.run.status_query",
|
||||
"generator_commands": [
|
||||
{
|
||||
"kind": "start",
|
||||
"action": {
|
||||
"kind": "handler",
|
||||
"id": "taskrun.restart"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "gen_greet_hallucinated_action",
|
||||
"category": "generation_consistency",
|
||||
"input": "你好呀",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.greet.hello",
|
||||
"generator_commands": [
|
||||
{
|
||||
"kind": "start",
|
||||
"action": {
|
||||
"kind": "workflow",
|
||||
"id": "ghost.flow"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "gen_chitchat_agree",
|
||||
"category": "generation_consistency",
|
||||
"input": "今天天气不错",
|
||||
"expect_branch": "fallback",
|
||||
"generator_commands": [
|
||||
{
|
||||
"kind": "chitchat"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"guard_domains": [
|
||||
"guard"
|
||||
],
|
||||
"protected_domains": [
|
||||
"guard"
|
||||
],
|
||||
"builtin_specs": [
|
||||
{
|
||||
"id": "guard.config_change",
|
||||
"domain": "guard",
|
||||
"version": 1,
|
||||
"priority": 1000,
|
||||
"matcher": {
|
||||
"includes_any": [
|
||||
"系统配置",
|
||||
"网关",
|
||||
"改密码"
|
||||
]
|
||||
},
|
||||
"risk": "high",
|
||||
"handler": "refuse.and_guide"
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user