Record the model A/B pipeline validation and the quota block
Why: Ruling 5 authorized the strong-model comparison with uncapped budget. The A/B infrastructure is proven end to end (qa7 on remote main, seeded claude-sonnet-4-6 via the user's existing dmxapi aggregator key, eval/live ran the full golden set), but the account holds CNY 0.49 against a CNY 0.62 per-request pre-deduction - all six turns failed with insufficient quota, captured honestly in the report. Evidence: SELF - .task/model-ab/ab-sonnet-quota-blocked.json + evidence row; smoke test of the dmxapi streaming endpoint succeeded before the run. Impact: Task blocked on user input only (top up dmxapi or provide another provider key). No code changes; worktree retained for immediate resumption.
This commit is contained in:
File diff suppressed because one or more lines are too long
@@ -0,0 +1,273 @@
|
||||
{
|
||||
"target": "http://127.0.0.1:18818",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"generatedAt": "2026-10-10T00:54:12Z",
|
||||
"tasks": [
|
||||
{
|
||||
"id": "minutes-basic",
|
||||
"turns": [
|
||||
{
|
||||
"user": "帮我总结一份会议纪要",
|
||||
"runId": "run_8907f5da5b7d",
|
||||
"status": "failed",
|
||||
"usage": {
|
||||
"id": "run_8907f5da5b7d",
|
||||
"status": "failed",
|
||||
"createdAt": "2026-10-10T00:54:12.788678154Z",
|
||||
"endedAt": "2026-10-10T00:56:53.130247413Z",
|
||||
"usage": {
|
||||
"promptTokens": 442455,
|
||||
"completionTokens": 6006,
|
||||
"totalTokens": 448461,
|
||||
"durationMs": 160340,
|
||||
"toolCalls": 0
|
||||
},
|
||||
"error": {
|
||||
"code": "run.failed",
|
||||
"message": "provider status 403: {\"error\":{\"message\":\"insufficient user quota (request id: 20261010005652819424504c14228f51XR3Ai7h)\",\"type\":\"dmx_api_error\",\"param\":\"\",\"code\":\"insufficient_user_quota\"}}"
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"toolCalls": 10,
|
||||
"toolFailures": 1,
|
||||
"fullRegens": 0,
|
||||
"apiDiscovery": 0,
|
||||
"readsOriginal": 1,
|
||||
"planUpdates": 2,
|
||||
"verifierChecks": 0
|
||||
},
|
||||
"issues": [
|
||||
"run 终态=failed"
|
||||
],
|
||||
"durationMs": 162188
|
||||
}
|
||||
],
|
||||
"total": {
|
||||
"toolCalls": 10,
|
||||
"toolFailures": 1,
|
||||
"fullRegens": 0,
|
||||
"apiDiscovery": 0,
|
||||
"readsOriginal": 1,
|
||||
"planUpdates": 2,
|
||||
"verifierChecks": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "plan-followup",
|
||||
"turns": [
|
||||
{
|
||||
"user": "帮我总结一份会议纪要",
|
||||
"runId": "run_3e588b3acda5",
|
||||
"status": "failed",
|
||||
"usage": {
|
||||
"id": "run_3e588b3acda5",
|
||||
"status": "failed",
|
||||
"createdAt": "2026-10-10T00:56:54.999291168Z",
|
||||
"endedAt": "2026-10-10T00:56:55.416959882Z",
|
||||
"usage": {
|
||||
"promptTokens": 0,
|
||||
"completionTokens": 0,
|
||||
"totalTokens": 0,
|
||||
"durationMs": 416,
|
||||
"toolCalls": 0
|
||||
},
|
||||
"error": {
|
||||
"code": "run.failed",
|
||||
"message": "provider status 403: {\"error\":{\"message\":\"insufficient user quota (request id: 20261010005655105260974c14228f50Czxx8KV)\",\"type\":\"dmx_api_error\",\"param\":\"\",\"code\":\"insufficient_user_quota\"}}"
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"toolCalls": 0,
|
||||
"toolFailures": 0,
|
||||
"fullRegens": 0,
|
||||
"apiDiscovery": 0,
|
||||
"readsOriginal": 0,
|
||||
"planUpdates": 0,
|
||||
"verifierChecks": 0
|
||||
},
|
||||
"issues": [
|
||||
"run 终态=failed"
|
||||
],
|
||||
"durationMs": 3015
|
||||
},
|
||||
{
|
||||
"user": "根据这个纪要，制定一个方案",
|
||||
"runId": "run_5489cca9189c",
|
||||
"status": "failed",
|
||||
"usage": {
|
||||
"id": "run_5489cca9189c",
|
||||
"status": "failed",
|
||||
"createdAt": "2026-10-10T00:56:58.010366407Z",
|
||||
"endedAt": "2026-10-10T00:56:58.412815147Z",
|
||||
"usage": {
|
||||
"promptTokens": 0,
|
||||
"completionTokens": 0,
|
||||
"totalTokens": 0,
|
||||
"durationMs": 401,
|
||||
"toolCalls": 0
|
||||
},
|
||||
"error": {
|
||||
"code": "run.failed",
|
||||
"message": "provider status 403: {\"error\":{\"message\":\"insufficient user quota (request id: 20261010005658116229887c14228f5fZFJecbf)\",\"type\":\"dmx_api_error\",\"param\":\"\",\"code\":\"insufficient_user_quota\"}}"
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"toolCalls": 0,
|
||||
"toolFailures": 0,
|
||||
"fullRegens": 0,
|
||||
"apiDiscovery": 0,
|
||||
"readsOriginal": 0,
|
||||
"planUpdates": 0,
|
||||
"verifierChecks": 0
|
||||
},
|
||||
"issues": [
|
||||
"run 终态=failed"
|
||||
],
|
||||
"durationMs": 3012
|
||||
},
|
||||
{
|
||||
"user": "方案太简要了，给我一个详细的方案和计划",
|
||||
"runId": "run_6f9390b47d94",
|
||||
"status": "failed",
|
||||
"usage": {
|
||||
"id": "run_6f9390b47d94",
|
||||
"status": "failed",
|
||||
"createdAt": "2026-10-10T00:57:01.021868203Z",
|
||||
"endedAt": "2026-10-10T00:57:01.393626511Z",
|
||||
"usage": {
|
||||
"promptTokens": 0,
|
||||
"completionTokens": 0,
|
||||
"totalTokens": 0,
|
||||
"durationMs": 371,
|
||||
"toolCalls": 0
|
||||
},
|
||||
"error": {
|
||||
"code": "run.failed",
|
||||
"message": "provider status 403: {\"error\":{\"message\":\"insufficient user quota (request id: 20261010005701124159262c14228f539OcuPSC)\",\"type\":\"dmx_api_error\",\"param\":\"\",\"code\":\"insufficient_user_quota\"}}"
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"toolCalls": 0,
|
||||
"toolFailures": 0,
|
||||
"fullRegens": 0,
|
||||
"apiDiscovery": 0,
|
||||
"readsOriginal": 0,
|
||||
"planUpdates": 0,
|
||||
"verifierChecks": 0
|
||||
},
|
||||
"issues": [
|
||||
"run 终态=failed"
|
||||
],
|
||||
"durationMs": 3010
|
||||
}
|
||||
],
|
||||
"total": {
|
||||
"toolCalls": 0,
|
||||
"toolFailures": 0,
|
||||
"fullRegens": 0,
|
||||
"apiDiscovery": 0,
|
||||
"readsOriginal": 0,
|
||||
"planUpdates": 0,
|
||||
"verifierChecks": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "qa-citation",
|
||||
"turns": [
|
||||
{
|
||||
"user": "这次会议最主要的事情是什么",
|
||||
"runId": "run_2a4f810f7f7d",
|
||||
"status": "failed",
|
||||
"usage": {
|
||||
"id": "run_2a4f810f7f7d",
|
||||
"status": "failed",
|
||||
"createdAt": "2026-10-10T00:57:04.067803899Z",
|
||||
"endedAt": "2026-10-10T00:57:04.303285404Z",
|
||||
"usage": {
|
||||
"promptTokens": 0,
|
||||
"completionTokens": 0,
|
||||
"totalTokens": 0,
|
||||
"durationMs": 234,
|
||||
"toolCalls": 0
|
||||
},
|
||||
"error": {
|
||||
"code": "run.failed",
|
||||
"message": "provider status 403: {\"error\":{\"message\":\"预扣费额度失败, 用户剩余额度: ¥0.491620, 需要预扣费额度: ¥0.621570 (request id: 20261010005704171163733c14228f5cukGTKzy)\",\"type\":\"dmx_api_error\",\"param\":\"\",\"code\":\"insufficient_user_quota\"}}"
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"toolCalls": 0,
|
||||
"toolFailures": 0,
|
||||
"fullRegens": 0,
|
||||
"apiDiscovery": 0,
|
||||
"readsOriginal": 0,
|
||||
"planUpdates": 0,
|
||||
"verifierChecks": 0
|
||||
},
|
||||
"issues": [
|
||||
"run 终态=failed"
|
||||
],
|
||||
"durationMs": 3017
|
||||
}
|
||||
],
|
||||
"total": {
|
||||
"toolCalls": 0,
|
||||
"toolFailures": 0,
|
||||
"fullRegens": 0,
|
||||
"apiDiscovery": 0,
|
||||
"readsOriginal": 0,
|
||||
"planUpdates": 0,
|
||||
"verifierChecks": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "minutes-pm-tool",
|
||||
"turns": [
|
||||
{
|
||||
"user": "生成会议纪要",
|
||||
"runId": "run_88b97836445f",
|
||||
"status": "failed",
|
||||
"usage": {
|
||||
"id": "run_88b97836445f",
|
||||
"status": "failed",
|
||||
"createdAt": "2026-10-10T00:57:07.104785963Z",
|
||||
"endedAt": "2026-10-10T00:57:07.322331234Z",
|
||||
"usage": {
|
||||
"promptTokens": 0,
|
||||
"completionTokens": 0,
|
||||
"totalTokens": 0,
|
||||
"durationMs": 217,
|
||||
"toolCalls": 0
|
||||
},
|
||||
"error": {
|
||||
"code": "run.failed",
|
||||
"message": "provider status 403: {\"error\":{\"message\":\"预扣费额度失败, 用户剩余额度: ¥0.491620, 需要预扣费额度: ¥0.580006 (request id: 20261010005707207307505c14228f559A506wI)\",\"type\":\"dmx_api_error\",\"param\":\"\",\"code\":\"insufficient_user_quota\"}}"
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"toolCalls": 0,
|
||||
"toolFailures": 0,
|
||||
"fullRegens": 0,
|
||||
"apiDiscovery": 0,
|
||||
"readsOriginal": 0,
|
||||
"planUpdates": 0,
|
||||
"verifierChecks": 0
|
||||
},
|
||||
"issues": [
|
||||
"run 终态=failed"
|
||||
],
|
||||
"durationMs": 3012
|
||||
}
|
||||
],
|
||||
"total": {
|
||||
"toolCalls": 0,
|
||||
"toolFailures": 0,
|
||||
"fullRegens": 0,
|
||||
"apiDiscovery": 0,
|
||||
"readsOriginal": 0,
|
||||
"planUpdates": 0,
|
||||
"verifierChecks": 0
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user