Agent-System/internal/jevdecide/eval_test.go

package jevdecide_test

import (
	"context"
	"encoding/json"
	"os"
	"path/filepath"
	"testing"
	"time"

	"chat-agent-system/internal/jevdecide"
)

type evalCase struct {
	ID    string `json:"id"`
	Suite string `json:"suite"`
	Input string `json:"input"`
	Gold  struct {
		Realtime *bool  `json:"realtime"`
		Bucket   string `json:"bucket"`
		Risk     string `json:"risk"`
	} `json:"gold"`
}

type evalFile struct {
	Cases []evalCase `json:"cases"`
}

type caseResult struct {
	ID         string  `json:"id"`
	Suite      string  `json:"suite"`
	Gold       any     `json:"gold"`
	Pred       any     `json:"pred"`
	Confidence float64 `json:"confidence"`
	Margin     float64 `json:"margin"`
	OK         bool    `json:"ok"`
	Error      string  `json:"error,omitempty"`
	LatencyMs  int64   `json:"latency_ms"`
}

// TestOfflineEvalAgainstLiveJev runs the ≥100-case corpus against a live jev-service.
// Gate (when RUN_JEV_EVAL=1): realtime accuracy≥85%, high-conf precision≥95%, profile agreement≥75%.
func TestOfflineEvalAgainstLiveJev(t *testing.T) {
	if os.Getenv("RUN_JEV_EVAL") != "1" {
		t.Skip("set RUN_JEV_EVAL=1 to run live jev offline evaluation")
	}
	raw, err := os.ReadFile(filepath.Join("testdata", "eval-v1.json"))
	if err != nil {
		t.Fatal(err)
	}
	var file evalFile
	if err := json.Unmarshal(raw, &file); err != nil {
		t.Fatal(err)
	}
	if len(file.Cases) < 100 {
		t.Fatalf("corpus size %d < 100", len(file.Cases))
	}

	client := jevdecide.NewClientFromEnv()
	ctx := context.Background()
	results := make([]caseResult, 0, len(file.Cases))

	var rtOK, rtN, rtHiOK, rtHiN, pfOK, pfN int

	for _, c := range file.Cases {
		start := time.Now()
		cr := caseResult{ID: c.ID, Suite: c.Suite, LatencyMs: 0}
		switch c.Suite {
		case "realtime":
			yes, ans, err := client.DecideRealtime(ctx, c.Input)
			cr.LatencyMs = time.Since(start).Milliseconds()
			cr.Gold = *c.Gold.Realtime
			if err != nil {
				cr.Error = err.Error()
				results = append(results, cr)
				rtN++
				continue
			}
			cr.Pred = yes
			cr.Confidence = ans.Confidence
			cr.Margin = ans.Margin()
			cr.OK = yes == *c.Gold.Realtime
			rtN++
			if cr.OK {
				rtOK++
			}
			if ans.Confidence >= 0.9 {
				rtHiN++
				// precision of high-conf subset: among conf>=0.9, how many correct
				if cr.OK {
					rtHiOK++
				}
			}
		case "profile":
			bucket, ans, err := client.DecideProfileBucket(ctx, c.Input)
			cr.LatencyMs = time.Since(start).Milliseconds()
			cr.Gold = c.Gold.Bucket
			if err != nil {
				cr.Error = err.Error()
				results = append(results, cr)
				pfN++
				continue
			}
			cr.Pred = bucket
			cr.Confidence = ans.Confidence
			cr.Margin = ans.Margin()
			cr.OK = bucket == c.Gold.Bucket
			pfN++
			if cr.OK {
				pfOK++
			}
		case "risk_ref":
			resp, err := client.Decide(ctx, c.Input, map[string]jevdecide.Question{
				"risk": {
					Type:     "choice",
					Question: "What document-change risk level does this describe?",
					Options:  []string{"low", "medium", "high", "critical"},
				},
			})
			cr.LatencyMs = time.Since(start).Milliseconds()
			cr.Gold = c.Gold.Risk
			if err != nil {
				cr.Error = err.Error()
				results = append(results, cr)
				continue
			}
			a := resp.Answers["risk"]
			cr.Pred = a.Choice
			cr.Confidence = a.Confidence
			cr.Margin = a.Margin()
			cr.OK = a.Choice == c.Gold.Risk
		default:
			t.Fatalf("unknown suite %s", c.Suite)
		}
		results = append(results, cr)
	}

	rtAcc := pct(rtOK, rtN)
	rtHiPrec := pct(rtHiOK, rtHiN)
	pfAcc := pct(pfOK, pfN)

	summary := map[string]any{
		"realtime": map[string]any{
			"n": rtN, "ok": rtOK, "accuracy": rtAcc,
			"high_conf_n": rtHiN, "high_conf_ok": rtHiOK, "high_conf_precision": rtHiPrec,
		},
		"profile": map[string]any{
			"n": pfN, "ok": pfOK, "agreement": pfAcc,
		},
		"gate": map[string]any{
			"realtime_accuracy_ge_85":     rtAcc >= 85.0,
			"realtime_hi_precision_ge_95": rtHiPrec >= 95.0 || rtHiN == 0,
			"profile_agreement_ge_75":     pfAcc >= 75.0,
		},
		"cases": results,
	}

	outDir := os.Getenv("JEV_EVAL_OUT")
	if outDir == "" {
		outDir = filepath.Join("..", "..", ".doc", "test-reports", "assets", "2026-09-24-jev-product-shadow")
	}
	_ = os.MkdirAll(outDir, 0o755)
	outPath := filepath.Join(outDir, "p0-offline-eval.json")
	blob, _ := json.MarshalIndent(summary, "", "  ")
	if err := os.WriteFile(outPath, blob, 0o644); err != nil {
		t.Fatalf("write report: %v", err)
	}
	t.Logf("wrote %s realtime_acc=%.1f hi_prec=%.1f profile_agr=%.1f", outPath, rtAcc, rtHiPrec, pfAcc)

	if rtAcc < 85.0 {
		t.Errorf("GATE FAIL realtime accuracy %.1f < 85", rtAcc)
	}
	if rtHiN > 0 && rtHiPrec < 95.0 {
		t.Errorf("GATE FAIL realtime high-conf precision %.1f < 95 (n=%d)", rtHiPrec, rtHiN)
	}
	if pfAcc < 75.0 {
		t.Errorf("GATE FAIL profile agreement %.1f < 75", pfAcc)
	}
}

func pct(ok, n int) float64 {
	if n == 0 {
		return 0
	}
	return 100.0 * float64(ok) / float64(n)
}

func TestCorpusSize(t *testing.T) {
	raw, err := os.ReadFile(filepath.Join("testdata", "eval-v1.json"))
	if err != nil {
		t.Fatal(err)
	}
	var file evalFile
	if err := json.Unmarshal(raw, &file); err != nil {
		t.Fatal(err)
	}
	if len(file.Cases) < 100 {
		t.Fatalf("need ≥100 cases, got %d", len(file.Cases))
	}
	t.Logf("corpus=%d", len(file.Cases))
}