weichenandCursor
932ba87daf
Generate 2–4 step L1 proposals with stable digests while leaving TopIntent and the legacy IntentPlan router unchanged. Co-authored-by: Cursor <cursoragent@cursor.com>
62 lines
2.0 KiB
JSON
62 lines
2.0 KiB
JSON
{
|
|
"version": "plan-proposal-eval-v1",
|
|
"notes": "U2 shadow counterfactual: would_execute is only allowed for independent low-risk read pairs. Denominator is the cases array length.",
|
|
"cases": [
|
|
{
|
|
"id": "query-then-query",
|
|
"input": "查一下项目风险，然后查相关法规",
|
|
"expect_combo": "read_only",
|
|
"expect_counterfactual": "would_execute",
|
|
"legacy_plan_steps": 2
|
|
},
|
|
{
|
|
"id": "query-then-generate",
|
|
"input": "查一下相关法规，然后帮我生成报价单",
|
|
"expect_combo": "needs_confirm",
|
|
"expect_counterfactual": "would_confirm",
|
|
"legacy_plan_steps": 0,
|
|
"hard_negative": true
|
|
},
|
|
{
|
|
"id": "internal-chain",
|
|
"input": "请查询相关法规并总结要点",
|
|
"expect_combo": "single",
|
|
"expect_counterfactual": "would_passthrough",
|
|
"legacy_plan_steps": 0
|
|
},
|
|
{
|
|
"id": "anaphora",
|
|
"input": "查一下相关法规，然后再看一下它的适用范围",
|
|
"expect_combo": "needs_clarify",
|
|
"expect_counterfactual": "would_clarify",
|
|
"hard_negative": true
|
|
},
|
|
{
|
|
"id": "security",
|
|
"input": "查一下相关法规，然后修改我的工号",
|
|
"expect_combo": "rejected",
|
|
"expect_counterfactual": "would_reject",
|
|
"hard_negative": true
|
|
},
|
|
{
|
|
"id": "batch-size-5",
|
|
"input": "1. 查法规\n2. 查案例\n3. 查进度\n4. 查权限\n5. 查文档状态",
|
|
"expect_combo": "rejected_batch_size",
|
|
"expect_counterfactual": "would_reject",
|
|
"hard_negative": true
|
|
},
|
|
{
|
|
"id": "numbered-3-query",
|
|
"input": "1. 查一下相关法规\n2. 查相似案例\n3. 查一下我的权限",
|
|
"expect_combo": "read_only",
|
|
"expect_counterfactual": "would_execute"
|
|
},
|
|
{
|
|
"id": "first-then-finally",
|
|
"input": "先查一下相关法规再查相似案例最后查一下我的权限",
|
|
"expect_combo": "read_only",
|
|
"expect_counterfactual": "would_execute"
|
|
}
|
|
]
|
|
}
|