weichen
db79b535ec
Why: I2 acceptance - the evaluator must emit metrics over a multiturn fixture no smaller than the existing corpus, not just print branch names. Evidence: go vet clean; full suite ok; intentctl eval prints precision=1.000 recall=1.000 (TP=12 FP=0 FN=0) over the 11-case multiturn fixture; MetricsOf unit test pins TP/FP/FN counting including wrong-match counting as both. Impact: CorpusMetrics + MetricsOf (expectation_hit counts as a hit); intentctl eval prints the metrics line after the summary; corpus_multiturn.json expanded to 11 cases (chained expectation, tie-break, ambiguous, fallback, handler action, clarify paths). Refs: Documents proposal 2026-10-07-agent-stack-n1q1i1-plans.md I2
238 lines
5.6 KiB
JSON
238 lines
5.6 KiB
JSON
{
|
|
"cases": [
|
|
{
|
|
"name": "report_fill_slot_two_turns",
|
|
"turns": [
|
|
{
|
|
"input": "帮我生成报告",
|
|
"expect_branch": "matched",
|
|
"expect_top": "report.generate",
|
|
"expect_commands": [
|
|
{
|
|
"kind": "start",
|
|
"action_kind": "handler",
|
|
"action_id": "reportHandler"
|
|
},
|
|
{
|
|
"kind": "clarify",
|
|
"missing": [
|
|
"topic"
|
|
],
|
|
"question": "要查询哪个主题？"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"input": "生成报告，主题是电机",
|
|
"expect_branch": "expectation_hit",
|
|
"expect_top": "report.generate",
|
|
"expect_commands": [
|
|
{
|
|
"kind": "set_slot",
|
|
"name": "topic",
|
|
"value": "电机"
|
|
},
|
|
{
|
|
"kind": "start",
|
|
"action_kind": "handler",
|
|
"action_id": "reportHandler",
|
|
"slots": {
|
|
"topic": "电机"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "ambiguous_then_tiebreak_match",
|
|
"turns": [
|
|
{
|
|
"input": "打结",
|
|
"expect_branch": "ambiguous",
|
|
"expect_commands": [
|
|
{
|
|
"kind": "clarify",
|
|
"candidates": [
|
|
"demo.tie.alpha",
|
|
"demo.tie.beta"
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"input": "你好呀",
|
|
"expect_branch": "matched",
|
|
"expect_top": "demo.greet.hello"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "single_matched_full_slots",
|
|
"turns": [
|
|
{
|
|
"input": "查询任务 42 的进度",
|
|
"expect_branch": "matched",
|
|
"expect_top": "demo.run.status_query",
|
|
"expect_commands": [
|
|
{
|
|
"kind": "start",
|
|
"action_kind": "handler",
|
|
"action_id": "taskrun.query_status",
|
|
"slots": {
|
|
"run_id": "42"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "single_fallback",
|
|
"turns": [
|
|
{
|
|
"input": "天气不错",
|
|
"expect_branch": "fallback",
|
|
"expect_commands": []
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "single_ambiguous_tie",
|
|
"turns": [
|
|
{
|
|
"input": "打结",
|
|
"expect_branch": "ambiguous",
|
|
"expect_commands": [
|
|
{
|
|
"kind": "clarify",
|
|
"candidates": [
|
|
"demo.tie.alpha",
|
|
"demo.tie.beta"
|
|
]
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "single_tiebreak_priority",
|
|
"turns": [
|
|
{
|
|
"input": "你好呀",
|
|
"expect_branch": "matched",
|
|
"expect_top": "demo.greet.hello"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "single_matched_handler_action",
|
|
"turns": [
|
|
{
|
|
"input": "查询任务 7 的进度",
|
|
"expect_branch": "matched",
|
|
"expect_top": "demo.run.status_query",
|
|
"expect_commands": [
|
|
{
|
|
"kind": "start",
|
|
"action_kind": "handler",
|
|
"action_id": "taskrun.query_status"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "two_turn_complete_then_fallback",
|
|
"turns": [
|
|
{
|
|
"input": "生成报告，主题是电机",
|
|
"expect_branch": "matched",
|
|
"expect_top": "report.generate",
|
|
"expect_commands": [
|
|
{
|
|
"kind": "start",
|
|
"action_kind": "handler",
|
|
"action_id": "reportHandler",
|
|
"slots": {
|
|
"topic": "电机"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"input": "天气不错",
|
|
"expect_branch": "fallback",
|
|
"expect_commands": []
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "two_turn_missing_then_other_intent",
|
|
"turns": [
|
|
{
|
|
"input": "帮我生成报告",
|
|
"expect_branch": "matched",
|
|
"expect_top": "report.generate"
|
|
},
|
|
{
|
|
"input": "查询任务 42 的进度",
|
|
"expect_branch": "matched",
|
|
"expect_top": "demo.run.status_query"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "single_clarify_missing_with_question",
|
|
"turns": [
|
|
{
|
|
"input": "帮我生成报告",
|
|
"expect_branch": "matched",
|
|
"expect_top": "report.generate",
|
|
"expect_commands": [
|
|
{
|
|
"kind": "start"
|
|
},
|
|
{
|
|
"kind": "clarify",
|
|
"missing": [
|
|
"topic"
|
|
],
|
|
"question": "要查询哪个主题？"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "two_turn_set_slot_pump",
|
|
"turns": [
|
|
{
|
|
"input": "帮我生成报告",
|
|
"expect_branch": "matched",
|
|
"expect_top": "report.generate"
|
|
},
|
|
{
|
|
"input": "生成报告，主题是水泵",
|
|
"expect_branch": "expectation_hit",
|
|
"expect_top": "report.generate",
|
|
"expect_commands": [
|
|
{
|
|
"kind": "set_slot",
|
|
"name": "topic",
|
|
"value": "水泵"
|
|
},
|
|
{
|
|
"kind": "start",
|
|
"action_kind": "handler",
|
|
"action_id": "reportHandler",
|
|
"slots": {
|
|
"topic": "水泵"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
]
|
|
} |