Agent-System-Dev/.task/genui-u1/score.py

#!/usr/bin/env python3
"""U1 评分器（原型）：聚合 results/ 各臂，输出指标表与成本核算。"""
import json
import os
from collections import Counter

HERE = os.path.dirname(os.path.abspath(__file__))
RESULTS = os.path.join(HERE, "results")
PRICE = {"MiniMax-M3": (0.3, 1.2), "MiniMax-M2.7-highspeed": (0.6, 2.4)}


def load(arm):
    d = os.path.join(RESULTS, arm)
    if not os.path.isdir(d):
        return []
    return [json.load(open(os.path.join(d, fn))) for fn in sorted(os.listdir(d)) if fn.endswith(".json")]


def cost_of(rec):
    pin, pout = PRICE.get(rec.get("model", ""), (0, 0))
    c = 0.0
    for x in rec.get("calls", []):
        u = x.get("usage", {})
        c += (u.get("prompt_tokens", 0) / 1e6 * pin + u.get("completion_tokens", 0) / 1e6 * pout) * 7.2
    return c


def tok(rec):
    p = sum(x.get("usage", {}).get("prompt_tokens", 0) for x in rec.get("calls", []))
    c = sum(x.get("usage", {}).get("completion_tokens", 0) for x in rec.get("calls", []))
    cached = sum(x.get("usage", {}).get("prompt_tokens_details", {}).get("cached_tokens", 0) for x in rec.get("calls", []))
    return p, c, cached


def arm_stats(recs, label):
    if not recs:
        return {"arm": label, "tasks": 0}
    p = sum(tok(r)[0] for r in recs); c = sum(tok(r)[1] for r in recs)
    cached = sum(tok(r)[2] for r in recs)
    cost = sum(cost_of(r) for r in recs)
    calls = sum(r.get("callCount", 1) for r in recs)
    wall = sum(r.get("wall_s", 0) for r in recs)
    out = {"arm": label, "tasks": len(recs), "calls": calls, "prompt_tok": p,
           "completion_tok": c, "cached_tok": cached, "cost_cny": round(cost, 2),
           "wall_s": round(wall, 1), "cost_cny_per_task": round(cost / len(recs), 3)}
    valid = [r for r in recs if r.get("valid_final")]
    if label.startswith("G"):
        vf = [r for r in recs if r.get("valid_first")]
        out.update({
            "valid_first": f"{len(vf)}/{len(recs)} ({100*len(vf)//len(recs)}%)",
            "valid_final": f"{len(valid)}/{len(recs)} ({100*len(valid)//len(recs)}%)",
            "repaired": sum(1 for r in recs if r.get("repaired")),
        })
        fits, src_cov, ids_provided = [], [], []
        for r in recs:
            fx = json.load(open(os.path.join(HERE, "fixtures.json")))
            task = next(t for t in fx["tasks"] if t["id"] == r["task"])
            types = set((r.get("stats_final") or r.get("stats_first") or {}).get("types", []) or [])
            if types:
                exp = set(task["expect"]["nodeTypesAny"])
                fits.append(len(types & exp) / len(exp))
            nsources = len(task["materials"])
            used = (r.get("stats_final") or r.get("stats_first") or {}).get("refsUsed") or []
            src_cov.append(len(set(used)) / nsources)
            if r.get("stats_final", {}).get("idsProvided"):
                ids_provided.append(1)
        out.update({
            "structure_fit": round(sum(fits) / len(fits), 2) if fits else None,
            "source_coverage": round(sum(src_cov) / len(src_cov), 2) if src_cov else None,
            "ids_provided_tasks": len(ids_provided),
        })
        # B 基线结构契合恒为 0（无结构选择能力），供对照
    if label == "B":
        md_tables = sum(1 for r in recs if r.get("goalKind") == "compare"
                        and any(l.strip().startswith("|") and "---" in l for l in r.get("markdown", "").splitlines()))
        cmp_tasks = sum(1 for r in recs if r.get("goalKind") == "compare")
        out["markdown_pipe_table_on_compare"] = f"{md_tables}/{cmp_tasks}"
    return out


def r1_divergence(recs_arm):
    """同料不同目标 → 结构是否分化：类型多重集 Jaccard 距离 + 主类型是否不同。"""
    fx = json.load(open(os.path.join(HERE, "fixtures.json")))
    groups = {}
    for t in fx["tasks"]:
        if t.get("sharedGroup"):
            groups.setdefault(t["sharedGroup"], []).append(t["id"])
    by_id = {r["task"]: r for r in recs_arm if r.get("valid_final")}
    rows = []
    for g, ids in sorted(groups.items()):
        ids = [i for i in ids if i in by_id]
        for a in range(len(ids)):
            for b in range(a + 1, len(ids)):
                ta = Counter(by_id[ids[a]]["stats_final"]["types"])
                tb = Counter(by_id[ids[b]]["stats_final"]["types"])
                inter = sum((ta & tb).values()); union = sum((ta | tb).values())
                jac = inter / union if union else 1.0
                prim_a = ta.most_common(1)[0][0] if ta else "-"
                prim_b = tb.most_common(1)[0][0] if tb else "-"
                rows.append({"group": g, "pair": f"{ids[a]}~{ids[b]}",
                             "jaccard": round(jac, 2), "primary": f"{prim_a}|{prim_b}",
                             "distinct": (prim_a != prim_b) or jac < 0.6})
    return rows


def main():
    fx = json.load(open(os.path.join(HERE, "fixtures.json")))
    summary = {}
    for arm in ("B", "G", "G-hs"):
        recs = load(arm)
        if recs:
            summary[arm] = arm_stats(recs, arm)
    g = load("G")
    if g:
        rows = r1_divergence(g)
        summary["r1_divergence"] = rows
        summary["r1_distinct_pairs"] = f"{sum(1 for r in rows if r['distinct'])}/{len(rows)}"
    # 按族拆 G 的结构契合
    fam = {}
    for r in g:
        if r.get("valid_final"):
            task = next(t for t in fx["tasks"] if t["id"] == r["task"])
            types = set(r["stats_final"]["types"])
            exp = set(task["expect"]["nodeTypesAny"])
            fam.setdefault(task["family"], []).append(len(types & exp) / len(exp))
    if fam:
        summary["G_structure_fit_by_family"] = {k: round(sum(v) / len(v), 2) for k, v in fam.items()}
    json.dump(summary, open(os.path.join(RESULTS, "summary.json"), "w"), ensure_ascii=False, indent=1)
    print(json.dumps(summary, ensure_ascii=False, indent=1))


if __name__ == "__main__":
    main()