Agent-Intent/cmd/intentctl/main_test.go

package main

import (
	"encoding/json"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"agent-intent/intent"
)

const testSpecJSON = `{
  "intents": [
    {"id": "demo.run.status_query", "domain": "demo", "version": 1,
     "matcher": {"includes_any": ["进度"], "excludes_any": ["配置"]},
     "slots": [{"key": "run_id", "pattern": "(?:run|任务)[:# ]*([A-Za-z0-9-]+)"}],
     "handler": "taskrun.query_status"},
    {"id": "demo.greet.hello", "domain": "demo", "version": 1, "matcher": {"includes_any": ["你好"]}},
    {"id": "demo.greet.hi", "domain": "demo", "version": 1, "matcher": {"includes_any": ["你好"]}}
  ]
}`

const testCorpusJSON = `{
  "cases": [
    {"name": "status", "input": "查询任务 42 的进度", "expect_branch": "matched", "expect_top": "demo.run.status_query", "expect_slots": {"run_id": "42"}},
    {"name": "ambiguous", "input": "你好", "expect_branch": "ambiguous"},
    {"name": "fallback", "input": "天气不错", "expect_branch": "fallback"}
  ]
}`

func writeTemp(t *testing.T, name, content string) string {
	t.Helper()
	dir := t.TempDir()
	if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644); err != nil {
		t.Fatal(err)
	}
	return dir
}

func TestLintExplainEval(t *testing.T) {
	dir := writeTemp(t, "specs.json", testSpecJSON)
	// 评测集放在配置目录之外：lint 的目录 glob 只认配置 overlay。
	corpusDir := t.TempDir()
	corpus := filepath.Join(corpusDir, "corpus.json")
	if err := os.WriteFile(corpus, []byte(testCorpusJSON), 0o644); err != nil {
		t.Fatal(err)
	}

	if err := run([]string{"lint", "-root", dir, "-protected", ""}); err != nil {
		t.Fatalf("lint: %v", err)
	}
	if err := run([]string{"explain", "-root", dir, "-protected", "", "查询任务 42 的进度"}); err != nil {
		t.Fatalf("explain: %v", err)
	}
	if err := run([]string{"eval", "-root", dir, "-protected", "", corpus}); err != nil {
		t.Fatalf("eval: %v", err)
	}
}

func TestLintRejectsInvalidConfig(t *testing.T) {
	dir := writeTemp(t, "specs.json", `{"intents": [{"id": "demo.x.one", "domain": "demo", "version": 1, "matcher": {}}]}`)
	if err := run([]string{"lint", "-root", dir}); err == nil {
		t.Fatal("expected lint failure on invalid spec")
	}
}

func TestEvalFailsOnWrongExpectation(t *testing.T) {
	dir := writeTemp(t, "specs.json", testSpecJSON)
	corpus := filepath.Join(t.TempDir(), "corpus.json")
	if err := os.WriteFile(corpus, []byte(`{"cases": [{"name": "wrong", "input": "你好", "expect_branch": "matched"}]}`), 0o644); err != nil {
		t.Fatal(err)
	}
	if err := run([]string{"eval", "-root", dir, "-protected", "", corpus}); err == nil {
		t.Fatal("expected eval failure on wrong expectation")
	}
}

func TestEvalRecordOnlyWritesMetrics(t *testing.T) {
	dir := writeTemp(t, "specs.json", testSpecJSON)
	tmp := t.TempDir()
	corpus := filepath.Join(tmp, "corpus.json")
	// 「不用查进度了」在否定修复后已正确回落，这里换成仍会误执行的输入来覆盖 record_only 计数。
	doc := `{"record_only": true, "cases": [
	  {"name": "miss", "category": "negation_reference", "input": "进度条的颜色真好看", "expect_branch": "fallback"},
	  {"name": "hit", "input": "天气不错", "expect_branch": "fallback"}]}`
	if err := os.WriteFile(corpus, []byte(doc), 0o644); err != nil {
		t.Fatal(err)
	}
	out := filepath.Join(tmp, "metrics.json")
	if err := run([]string{"eval", "-root", dir, "-protected", "", "-metrics-out", out, corpus}); err != nil {
		t.Fatalf("record_only eval should not fail: %v", err)
	}
	data, err := os.ReadFile(out)
	if err != nil {
		t.Fatal(err)
	}
	var s intent.EvalSummary
	if err := json.Unmarshal(data, &s); err != nil {
		t.Fatal(err)
	}
	if s.Schema != intent.EvalMetricsSchema || s.Overall.Cases != 2 || len(s.Categories) != 2 || s.Overall.MisExecutions != 1 {
		t.Fatalf("summary = %+v", s)
	}

	strict := filepath.Join(tmp, "strict.json")
	if err := os.WriteFile(strict, []byte(strings.Replace(doc, `"record_only": true, `, "", 1)), 0o644); err != nil {
		t.Fatal(err)
	}
	if err := run([]string{"eval", "-root", dir, "-protected", "", strict}); err == nil {
		t.Fatal("corpus without record_only must still fail on mismatch")
	}
}