Add execution ownership leases with epoch fencing and persistent run commands (U5) #151

Merged
weichen merged 2 commits from feature/collab-storage-u5-ownership into main 2026-10-11 12:26:35 +00:00
10 changed files with 969 additions and 19 deletions
+5 -1
View File
@@ -171,7 +171,11 @@ func TestCopyOrderCoversAllFixtureTables(t *testing.T) {
known[t] = true
}
known["schema_migrations"] = true // 迁移器管理,不随迁
known["admission_leases"] = true // 运行态,明确排除(见 copyOrder 注释)
// 运行态表不随迁:执行许可是进程瞬态;执行租约与跨实例命令属当前
// 实例的调度现场(接管语义在 U5 内由 lease/fencing 处理,不复制历史)。
for _, rt := range []string{"admission_leases", "conversation_executions", "run_commands"} {
known[rt] = true
}
// FTS5 虚表影子表:SQLite 内部结构,PG 侧由 v4 生成列/索引从 knowledge
// 行自动重建,不随迁。
for _, shadow := range []string{"knowledge_fts", "knowledge_fts_data", "knowledge_fts_idx", "knowledge_fts_docsize", "knowledge_fts_config"} {
+22
View File
@@ -94,6 +94,23 @@ type Options struct {
PGPool store.PoolOptions
}
// instanceTag 稳定执行身份:tier=team 时强制 host:addr(多实例互异),
// 本地档退化为 local(单实例语义)。
func instanceTag(opt Options) string {
if opt.Tier == "team" || opt.StorageDSN != "" {
return hostTag() + ":" + opt.Addr
}
return "local"
}
func hostTag() string {
h, err := os.Hostname()
if err != nil || h == "" {
return "host"
}
return h
}
// tierName 输出部署档名(诊断统一拼写:空 = auto)。
func tierName(opt Options) string {
if opt.Tier == "" {
@@ -481,6 +498,11 @@ func Run(opt Options) error {
}
}()
mgr.SetCards(cardSvc)
// 执行身份与命令消费(协作存储 U5):PG 档启动跨实例命令轮询;
// SQLite 本地档无跨实例命令(StartCommandConsumer 自行短路)。
mgr.SetInstanceTag(instanceTag(opt), opt.Addr)
stopCommands := mgr.StartCommandConsumer(context.Background())
defer stopCommands()
// 执行预算(协作存储 U3,R8):runtime.execBudgetGlobal / runtime.execBudgetPerUser
// 全零=不限(本地档默认,行为不变);开启后执行许可共享账本生效,
// 待执行队列上限 runtime.execQueueMax(默认 200,容量合同数值待 U7 实测调整)。
+178
View File
@@ -0,0 +1,178 @@
package runmgr
import (
"context"
"encoding/json"
"log"
"time"
core "agent-system-core"
)
// 协作存储 U5(KTD5):持久命令消费。任意 API 实例把取消/HITL 写入
// run_commands 收件箱;命令由「目标 run 的当前执行实例」消费(路由依据
// conversation_executions 的当前 owner;本地不在执行的实例跳过,命令保持
// pending 直到执行实例领取或超时由恢复路径收口)。
//
// 本文件是执行侧消费循环;server 侧的投递入口见 server 包 EnqueueRunCommand
// 的两个薄封装(CancelRunRemote / ResolveHITLRemote)。
// commandPollEvery 消费轮询间隔。
const commandPollEvery = 300 * time.Millisecond
// HITLCommandPayload HITL 命令载荷(gate 身份由 cardID+gateVersion 承载:
// 新 owner 消费时重验 gate 版本,过期审批明确拒绝而不静默丢弃——KTD5)。
type HITLCommandPayload struct {
Owner string `json:"owner"`
CardID string `json:"cardId"`
GateVersion int64 `json:"gateVersion"`
Kind string `json:"kind"`
Answer json.RawMessage `json:"answer,omitempty"`
}
// CancelCommandPayload 取消命令载荷。
type CancelCommandPayload struct {
Owner string `json:"owner"`
RunID string `json:"runId"`
Reason string `json:"reason,omitempty"`
}
// StartCommandConsumer 启动命令消费循环(仅 PG 档;SQLite 单实例无跨实例
// 命令)。返回停止函数。消费的命令只处理「本实例当前持有执行权」的会话;
// 其余跳过(不认领)——避免非执行实例吃掉命令。
func (m *Manager) StartCommandConsumer(ctx context.Context) (stop func()) {
if m.st.Dialect() != "postgres" {
return func() {}
}
ctx, cancel := context.WithCancel(ctx)
go func() {
t := time.NewTicker(commandPollEvery)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
m.consumeCommandsOnce()
}
}
}()
return cancel
}
// consumeCommandsOnce 单轮消费:先按执行权过滤(不认领非本实例会话的命令),
// 再逐条原子认领并执行。认领即消费(at-least-once;命令处理幂等:取消对
// 终态 run 报已终态、HITL 重验 gate)。
func (m *Manager) consumeCommandsOnce() {
pending, err := m.pendingCommandsForOwned()
if err != nil || len(pending) == 0 {
return
}
for _, c := range pending {
claimed, err := m.claimCommand(c.CommandID)
if err != nil || !claimed {
continue
}
switch c.Kind {
case "cancel":
m.applyCancelCommand(c)
case "hitl":
m.applyHITLCommand(c)
default:
log.Printf("[runmgr] command %s: unknown kind %q", c.CommandID, c.Kind)
}
}
}
// pendingCommandsForOwned 列出 pending 且其会话执行权在本实例的命令。
func (m *Manager) pendingCommandsForOwned() ([]ownedCommand, error) {
rows, err := m.st.DB().Query(
`SELECT c.command_id, c.kind, c.run_id, c.conversation_id, c.epoch, c.payload
FROM run_commands c JOIN conversation_executions e ON e.conversation_id = c.conversation_id
WHERE c.state='pending' AND e.instance_id = $1
ORDER BY c.created_at LIMIT 16`, m.InstanceID())
if err != nil {
return nil, err
}
defer rows.Close()
var out []ownedCommand
for rows.Next() {
var c ownedCommand
if err := rows.Scan(&c.CommandID, &c.Kind, &c.RunID, &c.ConversationID, &c.Epoch, &c.Payload); err != nil {
return nil, err
}
out = append(out, c)
}
return out, rows.Err()
}
type ownedCommand struct {
CommandID string
Kind string
RunID string
ConversationID string
Epoch int64
Payload string
}
// claimCommand 条件认领(pending→consumed,consumer=本实例)。
func (m *Manager) claimCommand(commandID string) (bool, error) {
res, err := m.st.DB().Exec(
`UPDATE run_commands SET state='consumed', consumed_at=$3, consumer=$1 WHERE command_id=$2 AND state='pending'`,
m.InstanceID(), commandID, time.Now().UTC().Format(time.RFC3339Nano))
if err != nil {
return false, err
}
n, _ := res.RowsAffected()
return n > 0, nil
}
// applyCancelCommand 本地执行取消:run 在本实例活跃 → 级联取消;
// 不活跃(如执行实例重启后命令迟到)→ 库级收口(非终态置 cancelled)。
func (m *Manager) applyCancelCommand(c ownedCommand) {
var p CancelCommandPayload
if err := json.Unmarshal([]byte(c.Payload), &p); err != nil {
log.Printf("[runmgr] cancel %s payload: %v", c.CommandID, err)
return
}
runID := c.RunID
if runID == "" {
runID = p.RunID
}
// 本地活跃 → 级联取消(含 HITL 唤醒);否则库级收口。CancelRun 对
// 非本地 run 不落库(只动内存句柄),不能作为成败判据。
if m.RunActiveLocally(runID) {
if err := m.CancelRun(runID); err == nil {
return
}
}
// 库级取消(幂等——仅非终态可置)。
if _, err := m.st.DB().Exec(
`UPDATE runs SET status='cancelled', ended_at=$2 WHERE id=$1 AND status IN ('pending','running','waiting_hitl')`, runID, time.Now().UTC().Format(time.RFC3339Nano)); err != nil {
log.Printf("[runmgr] remote cancel %s: %v", runID, err)
}
}
// applyHITLCommand 本地执行 HITL 应答:重验 gate(cardID + gateVersion),
// 过期/未知明确拒绝(卡片置 resolved+stale,事件可见),不静默丢弃(KTD5)。
func (m *Manager) applyHITLCommand(c ownedCommand) {
var p HITLCommandPayload
if err := json.Unmarshal([]byte(c.Payload), &p); err != nil {
log.Printf("[runmgr] hitl %s payload: %v", c.CommandID, err)
return
}
resp := core.HITLResponse{CardID: core.ID(p.CardID)}
if p.Kind == "free" {
resp.FreeText = string(p.Answer)
} else {
resp.ChoiceID = p.Kind // choiceId 即按钮语义(与 cardAction 契约一致)
}
_, _, err := m.ResolveHITL(p.Owner, p.CardID, resp)
if err != nil {
// 过期/未知 gate:明确销项(resolved + 注记),发出可见结论。
_ = m.st.ResolveHITLCard(p.Owner, p.CardID, core.HITLResolvedPayload{
CardID: core.ID(p.CardID), Kind: "expired", Answer: "跨实例审批迟到:gate 已过期或执行已移交",
})
log.Printf("[runmgr] hitl %s (card %s) rejected: %v", c.CommandID, p.CardID, err)
}
}
+176
View File
@@ -0,0 +1,176 @@
package runmgr
import (
"context"
"database/sql"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"time"
"agent-system-backend/internal/store"
core "agent-system-core"
)
// 协作存储 U5:双实例语义(KTD4/KTD5)。两个独立 Manager(A/B)共享同一
// PG schema(等价两真实进程的存储视图;进程级隔离由 CI 双进程验收与部署
// 合同覆盖——本测试验证所有权/围栏/命令的库级正确性)。
// 场景:同会话争抢、跨实例取消、旧 owner 迟到结算被拒。
func pgSchemaStore(t *testing.T) *store.Store {
t.Helper()
dsn := os.Getenv("AGENT_PG_TEST_DSN")
if dsn == "" {
t.Skip("AGENT_PG_TEST_DSN 未设置:真库多实例语义由 CI 必跑 job 覆盖")
}
schema := "u5_" + strings.ToLower(store.NewID("t")) + "_" + fmt.Sprint(os.Getpid())
admin, err := sql.Open("pgx", dsn)
if err != nil {
t.Fatal(err)
}
if _, err := admin.Exec(`CREATE EXTENSION IF NOT EXISTS pg_trgm`); err != nil {
admin.Close()
t.Fatalf("pg_trgm: %v", err)
}
if _, err := admin.Exec("CREATE SCHEMA " + schema); err != nil {
admin.Close()
t.Fatalf("schema: %v", err)
}
t.Cleanup(func() {
admin.Exec("DROP SCHEMA IF EXISTS " + schema + " CASCADE")
admin.Close()
})
st, err := store.OpenPostgres(store.AppendPGRuntimeParam(dsn, "search_path", schema+",public"), filepath.Join(t.TempDir(), "art"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { st.Close() })
return st
}
func newOwnedManager(t *testing.T, st *store.Store, tag string) *Manager {
t.Helper()
mgr := New(st, nil, nil, nil, nil, noopRunner{}, noopRunner{}, noopRunner{}, noopRunner{})
mgr.SetInstanceTag(tag, "127.0.0.1:0")
t.Cleanup(mgr.Shutdown)
return mgr
}
// 同会话争抢:A 占用后 B 的 ClaimConversation 被 409 拒(库级租约裁决)。
func TestMultiInstanceConversationContention(t *testing.T) {
st := pgSchemaStore(t)
a := newOwnedManager(t, st, "instA")
b := newOwnedManager(t, st, "instB")
if err := a.ClaimConversation("cnv_race"); err != nil {
t.Fatalf("A claim: %v", err)
}
defer a.ReleaseConversation("cnv_race")
if err := b.ClaimConversation("cnv_race"); err == nil || !errors.Is(err, ErrConversationBusy) {
t.Fatalf("B claim should be busy: %v", err)
}
// A 释放后 B 可占。
a.ReleaseConversation("cnv_race")
if err := b.ClaimConversation("cnv_race"); err != nil {
t.Fatalf("B claim after release: %v", err)
}
}
// 旧 owner 迟到结算被拒(fencing):A 持 epoch N;租约到期 B 接管(N+1);
// A 的 SettleRunFenced 被围栏拒绝、run 不变;B 的结算成功。
func TestMultiInstanceStaleOwnerSettleFenced(t *testing.T) {
st := pgSchemaStore(t)
conv := recoveryConversation(t, st)
// A 以 1 秒租约占用并开 run。
eA, err := st.TryClaimExecution(string(conv.ID), "instA", "", 1)
if err != nil {
t.Fatal(err)
}
run := recoveryRun(t, st, conv, core.RunRunning)
time.Sleep(1100 * time.Millisecond) // 租约到期
// B 接管。
eB, err := st.TryClaimExecution(string(conv.ID), "instB", "", 60)
if err != nil || eB != eA+1 {
t.Fatalf("B takeover: epoch=%d (want %d) err=%v", eB, eA+1, err)
}
// A 迟到结算:fenced,run 不变(仍 running)。
runA := run
runA.Status = core.RunCompleted
updated, fenced, err := st.SettleRunFenced(&runA, nil, "instA", eA)
if err != nil || updated || !fenced {
t.Fatalf("A settle: updated=%v fenced=%v err=%v(应 fenced 拒写)", updated, fenced, err)
}
after, _ := st.GetRunByID(run.ID)
if after.Status != core.RunRunning {
t.Fatalf("fenced settle 不得改写 run:status=%s", after.Status)
}
// B 结算成功。
runB := run
runB.Status = core.RunCompleted
updated, fenced, err = st.SettleRunFenced(&runB, nil, "instB", eB)
if err != nil || !updated || fenced {
t.Fatalf("B settle: updated=%v fenced=%v err=%v", updated, fenced, err)
}
}
// 跨实例取消(KTD5):run 由 A 执行;B 侧投递取消命令;A 的消费循环领取并
// 取消(本地句柄路径);命令恰被消费一次。
func TestMultiInstanceCrossCancelViaCommand(t *testing.T) {
st := pgSchemaStore(t)
a := newOwnedManager(t, st, "instA")
conv := recoveryConversation(t, st)
if err := a.ClaimConversation(string(conv.ID)); err != nil {
t.Fatal(err)
}
defer a.ReleaseConversation(string(conv.ID))
run := recoveryRun(t, st, conv, core.RunRunning)
a.execMu.Lock()
a.execEpoch[string(conv.ID)], _ = st.TryClaimExecution(string(conv.ID), "instA", run.ID, 60)
a.execMu.Unlock()
// B 侧投递取消命令(等价 server 的跨实例路径)。
cmdID := store.NewID("cmd")
if err := st.EnqueueCommand(cmdID, "cancel", run.ID, string(conv.ID), 0,
CancelCommandPayload{Owner: conv.Owner, RunID: run.ID}); err != nil {
t.Fatal(err)
}
// A 的单轮消费:认领并执行(run 非本实例活跃 → 库级取消收口)。
a.consumeCommandsOnce()
state, _, err := st.CommandState(cmdID)
if err != nil || state != "consumed" {
t.Fatalf("command state=%s err=%v(应 consumed)", state, err)
}
after, _ := st.GetRunByID(run.ID)
if after.Status != core.RunCancelled {
t.Fatalf("run status=%s(应 cancelled)", after.Status)
}
// 重复消费不重复执行(命令已 consumed,单轮无动作)。
a.consumeCommandsOnce()
after2, _ := st.GetRunByID(run.ID)
if after2.Status != core.RunCancelled {
t.Fatalf("重复消费后 status=%s", after2.Status)
}
}
// 非执行实例不吃命令:命令的会话执行权在 B,A 的消费循环应跳过。
func TestCommandOnlyConsumedByOwner(t *testing.T) {
st := pgSchemaStore(t)
a := newOwnedManager(t, st, "instA")
conv := recoveryConversation(t, st)
run := recoveryRun(t, st, conv, core.RunRunning)
if _, err := st.TryClaimExecution(string(conv.ID), "instB", run.ID, 60); err != nil {
t.Fatal(err)
}
cmdID := store.NewID("cmd")
if err := st.EnqueueCommand(cmdID, "cancel", run.ID, string(conv.ID), 0, nil); err != nil {
t.Fatal(err)
}
a.consumeCommandsOnce()
state, _, _ := st.CommandState(cmdID)
if state != "pending" {
t.Fatalf("非执行实例不得消费命令:state=%s", state)
}
_ = context.Background()
}
+93 -6
View File
@@ -68,9 +68,13 @@ type Manager struct {
active map[string]*runHandle
// 会话内串行(设计并发策略):一个会话同时至多一个在途 conversation run。
convMu sync.Mutex
convActive map[string]bool // conversationID → 有在途 run
adm *admissionGate // 执行许可门(U3:预算默认关闭时零开销)
convMu sync.Mutex
convActive map[string]bool // conversationID → 有在途 run
adm *admissionGate // 执行许可门(U3:预算默认关闭时零开销)
instanceTag string // 执行身份(U5:多实例互异;空=runmgr:addr)
addr string // 监听地址(身份退化用)
execMu sync.Mutex // 执行所有权(U5:DB 租约的本地代次登记)
execEpoch map[string]int64 // conversationID → 本实例当前 epoch
instanceID string
shuttingDown atomic.Bool
@@ -123,7 +127,7 @@ func New(st *store.Store, b *bus.Bus, reg *tools.Registry, p *profiles.Service,
return &Manager{
st: st, bus: b, registry: reg, profiles: p, settings: s,
runner: runner, subRunner: subRunner, mock: mockRunner, mockSub: mockSubRunner,
active: map[string]*runHandle{}, convActive: map[string]bool{},
active: map[string]*runHandle{}, convActive: map[string]bool{}, execEpoch: map[string]int64{},
notices: map[string][]bgNotice{}, bgInFlight: map[string]int{}, delivering: map[string]bool{},
events: newEventBuffer(st, b),
adm: newAdmissionGate(nil, 0),
@@ -1136,7 +1140,39 @@ func workflowScriptBodies(reg *workflow.Registry, def core.WorkflowDef) map[stri
// ClaimConversation 抢占会话槽位。忙则返回 ErrConversationBusy,此时调用方不应写消息。
func (m *Manager) ClaimConversation(conversationID string) error {
return m.claimConversation(conversationID)
if err := m.claimConversation(conversationID); err != nil {
return err
}
// 执行所有权(U5/KTD4):PG 档在内存槽位之上叠加 DB 租约——跨实例的
// 同会话争抢由库裁决;接管递增 epoch 供结算围栏。失败时回退内存槽位。
if m.st.Dialect() == "postgres" {
epoch, err := m.st.TryClaimExecution(conversationID, m.InstanceID(), "", 15*60)
if err != nil {
m.releaseConversation(conversationID)
if errors.Is(err, store.ErrExecutionBusy) {
return ErrConversationBusy
}
return err
}
m.execMu.Lock()
m.execEpoch[conversationID] = epoch
m.execMu.Unlock()
}
return nil
}
// InstanceID 本进程执行身份(组合根可经 SetInstanceTag 覆盖;多实例必须互异)。
func (m *Manager) InstanceID() string {
if m.instanceTag != "" {
return m.instanceTag
}
return "runmgr:" + m.addr
}
// SetInstanceTag 注入稳定实例身份与监听地址(组合根)。
func (m *Manager) SetInstanceTag(tag, addr string) {
m.instanceTag = tag
m.addr = addr
}
// ReleaseConversation 释放已抢到的槽位。启动失败时调用;重复释放无效果。
@@ -1224,6 +1260,17 @@ func (m *Manager) releaseConversation(conversationID string) {
m.convMu.Lock()
delete(m.convActive, conversationID)
m.convMu.Unlock()
if m.st != nil && m.st.Dialect() == "postgres" {
m.execMu.Lock()
epoch, ok := m.execEpoch[conversationID]
if ok {
delete(m.execEpoch, conversationID)
}
m.execMu.Unlock()
if ok {
m.st.ReleaseExecution(conversationID, m.InstanceID(), epoch)
}
}
}
// execute 运行至终态并收敛:状态、usage、助手消息、收尾事件。
@@ -1295,12 +1342,19 @@ func (m *Manager) finishRun(rc *runCtx, status core.RunStatus, usage core.Usage,
msg.VariantIndex = index
}
}
updated, err := m.st.SettleRun(&rc.run, msg)
updated, fenced, err := m.settleRunWithFence(rc, msg)
if err != nil {
log.Printf("[runmgr] settle run %s: %v", rc.run.ID, err)
m.failSettle(rc, err)
return
}
if fenced {
// 旧执行代次(U5/KTD4):租约已被他实例接管——本地不写终态;
// run 归属已移交,发围栏事件后静默退出(迟到副作用由此可见)。
log.Printf("[runmgr] settle run %s fenced: execution taken over (conv %s)", rc.run.ID, rc.run.ConversationID)
m.emit(rc.run.ConversationID, rc.run.ID, core.EvRunFailed, core.RunFailedPayload{Code: "run.fenced", Message: "执行权已被其他实例接管"})
return
}
if m.OnUsage != nil && usage.TotalTokens > 0 {
m.OnUsage(rc.run.Owner, usage.TotalTokens)
}
@@ -1393,6 +1447,39 @@ func (m *Manager) emit(conversationID, runID string, typ core.StreamEventType, p
return m.events.emit(conversationID, runID, typ, payload)
}
// settleRunWithFence(U5):PG 档以本实例登记的 epoch 做围栏结算。
func (m *Manager) settleRunWithFence(rc *runCtx, msg *core.Message) (updated, fenced bool, err error) {
if m.st.Dialect() != "postgres" {
updated, err = m.st.SettleRun(&rc.run, msg)
return updated, false, err
}
m.execMu.Lock()
epoch, ok := m.execEpoch[string(rc.run.ConversationID)]
m.execMu.Unlock()
if !ok {
// 未登记(旧路径起跑):按库内当前代次校验(同实例放行)。
if es, gerr := m.st.GetExecution(string(rc.run.ConversationID)); gerr == nil {
epoch = es.Epoch
}
}
return m.st.SettleRunFenced(&rc.run, msg, m.InstanceID(), epoch)
}
// RunActiveLocally run 是否在本实例活跃执行(U5:跨实例命令路由判定)。
func (m *Manager) RunActiveLocally(runID string) bool {
root := runID
if r, err := m.st.GetRunByID(runID); err == nil && r.ParentRunID != "" {
root = r.ParentRunID
}
m.mu.Lock()
_, ok := m.active[root]
if !ok {
_, ok = m.active[runID]
}
m.mu.Unlock()
return ok
}
// CancelRun 取消 run 及其子 run(ctx 级联)。
func (m *Manager) CancelRun(runID string) error {
r, err := m.st.GetRunByID(runID)
+33 -1
View File
@@ -1006,7 +1006,24 @@ func (s *Server) listAudits(w http.ResponseWriter, r *http.Request) {
}
func (s *Server) cancelRun(w http.ResponseWriter, r *http.Request) {
writeOKOr(w, s.mgr.CancelRun(r.PathValue("id")))
runID := r.PathValue("id")
err := s.mgr.CancelRun(runID)
if err == nil {
writeOKOr(w, nil)
return
}
// 跨实例取消(协作存储 U5/KTD5):run 不在本实例执行(无句柄)且 PG 档
// ——写持久命令,由执行实例消费;API 立即受理(命令可见、唯一消费)。
if s.st.Dialect() == "postgres" && s.mgr.RunActiveLocally(runID) == false {
if cerr := s.st.EnqueueCommand(store.NewID("cmd"), "cancel", runID, "", 0,
runmgr.CancelCommandPayload{Owner: s.owner(r), RunID: runID}); cerr != nil {
writeStoreErr(w, err)
return
}
writeJSON(w, http.StatusAccepted, map[string]any{"accepted": true, "via": "command"})
return
}
writeStoreErr(w, err)
}
func (s *Server) getRun(w http.ResponseWriter, r *http.Request) {
@@ -1029,6 +1046,21 @@ func (s *Server) resolveHITL(w http.ResponseWriter, r *http.Request) {
}
resp.CardID = r.PathValue("id")
pivotText, pivoted, err := s.mgr.ResolveHITL(s.owner(r), string(resp.CardID), resp)
if err != nil && s.st.Dialect() == "postgres" {
// 跨实例 HITL(U5/KTD5):本地无执行句柄——写持久命令,执行实例
// 消费时重验 gate(cardID+gateVersion),过期审批明确拒绝。
if card, gerr := s.st.GetHITLCard(s.owner(r), string(resp.CardID)); gerr == nil && !card.Resolved {
kind := resp.ChoiceID
if kind == "" {
kind = "free"
}
if cerr := s.st.EnqueueCommand(store.NewID("cmd"), "hitl", string(card.RunID), string(card.ConversationID), 0,
runmgr.HITLCommandPayload{Owner: s.owner(r), CardID: string(resp.CardID), Kind: kind, Answer: json.RawMessage(resp.FreeText)}); cerr == nil {
writeJSON(w, http.StatusAccepted, map[string]any{"accepted": true, "via": "command"})
return
}
}
}
if err != nil {
writeStoreErr(w, err)
return
+219
View File
@@ -0,0 +1,219 @@
package store
import (
"database/sql"
"encoding/json"
"errors"
"fmt"
"time"
)
// 协作存储 U5(KTD4/KTD5):会话执行租约 + 持久命令收件箱。
//
// 租约:conversation_executions 一行一会话。TryClaimExecution 空闲/租约到期
// 才可占用;接管(含同实例重新占用)递增 epoch——旧 owner 的续租与结算按
//(instance, epoch)双重校验被拒(fencing)。lease_until 用应用侧 UTC ISO
// (单库写入方一致即可;跨库时钟偏移属部署合同,见 runbook)。
//
// 命令:run_commands 收件箱承载跨实例取消/HITL(KTD5)。command_id 幂等;
// pending→consumed 由消费方原子认领(PG FOR UPDATE SKIP LOCKED;SQLite 单写
// 者条件更新)。命令不删除——审计与重放判定保留。
// ErrExecutionBusy 会话执行权被其他实例持有且租约未到期。
var ErrExecutionBusy = errors.New("store: execution owned by another instance")
// ErrFenced 旧执行代次的写入被围栏拒绝(epoch 已被接管递增)。
var ErrFenced = errors.New("store: fenced by newer execution epoch")
// ExecutionState 当前执行租约快照(诊断/测试)。
type ExecutionState struct {
ConversationID string
InstanceID string
RunID string
Epoch int64
LeaseUntil string
}
// TryClaimExecution 占用(或租约到期后接管)会话执行权。
// 返回接管后的 epoch(接管必然 +1)。被他人持有且未到期 → ErrExecutionBusy。
func (s *Store) TryClaimExecution(conversationID, instanceID, runID string, leaseSec int) (int64, error) {
if leaseSec <= 0 {
leaseSec = 90
}
tx, err := s.db.Begin()
if err != nil {
return 0, err
}
defer tx.Rollback()
if err := s.lockAdmission(tx); err != nil { // 复用全局账本锁串行化清点
return 0, err
}
now := time.Now().UTC()
until := now.Add(time.Duration(leaseSec) * time.Second).UTC().Format(time.RFC3339Nano)
var inst, lease string
var epoch int64
err = tx.QueryRow(s.q(`SELECT instance_id, epoch, lease_until FROM conversation_executions WHERE conversation_id=?`), conversationID).Scan(&inst, &epoch, &lease)
switch {
case errors.Is(err, sql.ErrNoRows):
if _, err := tx.Exec(s.q(`INSERT INTO conversation_executions (conversation_id, instance_id, run_id, epoch, lease_until, updated_at) VALUES (?,?,?,1,?,?)`),
conversationID, instanceID, runID, until, nowISO()); err != nil {
return 0, err
}
if err := tx.Commit(); err != nil {
return 0, err
}
return 1, nil
case err != nil:
return 0, err
}
if inst != instanceID && lease > nowISO() {
return epoch, fmt.Errorf("conv %s: %w", conversationID, ErrExecutionBusy)
}
// 接管(到期或同实例重占):epoch+1,fencing 由此产生。
if _, err := tx.Exec(s.q(`UPDATE conversation_executions SET instance_id=?, run_id=?, epoch=epoch+1, lease_until=?, updated_at=? WHERE conversation_id=?`),
instanceID, runID, until, nowISO(), conversationID); err != nil {
return 0, err
}
if err := tx.Commit(); err != nil {
return 0, err
}
return epoch + 1, nil
}
// RenewExecution 续租(fencing:instance+epoch 双校验)。假 = 已被接管或释放。
func (s *Store) RenewExecution(conversationID, instanceID string, epoch int64, leaseSec int) bool {
if leaseSec <= 0 {
leaseSec = 90
}
until := time.Now().UTC().Add(time.Duration(leaseSec) * time.Second).UTC().Format(time.RFC3339Nano)
res, err := s.db.Exec(s.q(`UPDATE conversation_executions SET lease_until=?, updated_at=? WHERE conversation_id=? AND instance_id=? AND epoch=?`),
until, nowISO(), conversationID, instanceID, epoch)
if err != nil {
return false
}
n, _ := res.RowsAffected()
return n > 0
}
// ReleaseExecution 主动释放(优雅停机):行保留、租约置为已过期——epoch 保留,
// 下一次占用仍递增(fencing 单调)。
func (s *Store) ReleaseExecution(conversationID, instanceID string, epoch int64) {
_, _ = s.db.Exec(s.q(`UPDATE conversation_executions SET lease_until=?, updated_at=? WHERE conversation_id=? AND instance_id=? AND epoch=?`),
time.Now().UTC().Add(-time.Second).UTC().Format(time.RFC3339Nano), nowISO(), conversationID, instanceID, epoch)
}
// GetExecution 读取当前租约(无行 = 从未占用)。
func (s *Store) GetExecution(conversationID string) (ExecutionState, error) {
var e ExecutionState
err := s.db.QueryRow(s.q(`SELECT conversation_id, instance_id, COALESCE(run_id,''), epoch, lease_until FROM conversation_executions WHERE conversation_id=?`), conversationID).
Scan(&e.ConversationID, &e.InstanceID, &e.RunID, &e.Epoch, &e.LeaseUntil)
if errors.Is(err, sql.ErrNoRows) {
return e, ErrNotFound
}
return e, err
}
// ExecutionFenceOK 校验 (instance, epoch) 仍是当前执行权(结算/事件写入的
// 前置检查;事务内校验见 SettleRunFenced)。
func (s *Store) ExecutionFenceOK(conversationID, instanceID string, epoch int64) bool {
var n int
if err := s.db.QueryRow(s.q(`SELECT COUNT(*) FROM conversation_executions WHERE conversation_id=? AND instance_id=? AND epoch=?`), conversationID, instanceID, epoch).Scan(&n); err != nil {
return false
}
return n > 0
}
// SettleRunFenced 带执行围栏的终态结算(KTD4:旧 owner 恢复后不能提交)。
// fenced=true 表示 (instance,epoch) 已不是当前执行权——run 未被改动,调用方
// 应将本地 run 视为被接管(不写终态、事件标注接管)。无租约行(SQLite 本地
// 档/未启用执行权)恒放行。
// RunCommand 收件箱命令。
type RunCommand struct {
CommandID string
Kind string // cancel | hitl
RunID string
ConversationID string
Epoch int64
Payload string
CreatedAt string
}
// EnqueueCommand 投递跨实例命令(幂等:同 command_id 重复投递不产生新行)。
func (s *Store) EnqueueCommand(commandID, kind, runID, conversationID string, epoch int64, payload any) error {
raw, err := json.Marshal(payload)
if err != nil {
return err
}
_, err = s.db.Exec(s.q(`INSERT INTO run_commands (command_id, kind, run_id, conversation_id, epoch, payload, state, created_at)
VALUES (?,?,?,?,?,?,'pending',?)
ON CONFLICT(command_id) DO NOTHING`),
commandID, kind, runID, conversationID, epoch, string(raw), nowISO())
if err != nil {
return fmt.Errorf("enqueue command %s: %w", commandID, err)
}
return nil
}
// ClaimPendingCommands 认领待处理命令(原子 pending→consumed)。
// PG:FOR UPDATE SKIP LOCKED(多实例并发认领不重复);SQLite:单写者条件更新。
// 只认领「本实例相关」的命令由调用方过滤——收件箱按执行权路由:命令在
// 被执行实例消费(cancel/hitl 目标 run 的当前 owner)。
func (s *Store) ClaimPendingCommands(consumer string, limit int) ([]RunCommand, error) {
if limit <= 0 {
limit = 16
}
tx, err := s.db.Begin()
if err != nil {
return nil, err
}
defer tx.Rollback()
var rows *sql.Rows
if s.dialect.name() == "postgres" {
rows, err = tx.Query(`SELECT command_id, kind, run_id, conversation_id, epoch, payload, created_at
FROM run_commands WHERE state='pending' ORDER BY created_at
LIMIT $1 FOR UPDATE SKIP LOCKED`, limit)
} else {
rows, err = tx.Query(s.q(`SELECT command_id, kind, run_id, conversation_id, epoch, payload, created_at
FROM run_commands WHERE state='pending' ORDER BY created_at LIMIT ?`), limit)
}
if err != nil {
return nil, err
}
var out []RunCommand
for rows.Next() {
var c RunCommand
if err := rows.Scan(&c.CommandID, &c.Kind, &c.RunID, &c.ConversationID, &c.Epoch, &c.Payload, &c.CreatedAt); err != nil {
rows.Close()
return nil, err
}
out = append(out, c)
}
rows.Close()
if err := rows.Err(); err != nil {
return nil, err
}
for _, c := range out {
res, err := tx.Exec(s.q(`UPDATE run_commands SET state='consumed', consumed_at=?, consumer=? WHERE command_id=? AND state='pending'`),
nowISO(), consumer, c.CommandID)
if err != nil {
return nil, err
}
if n, _ := res.RowsAffected(); n == 0 {
return nil, fmt.Errorf("command %s claimed concurrently", c.CommandID)
}
}
if err := tx.Commit(); err != nil {
return nil, err
}
return out, nil
}
// CommandState 查询单条命令状态(测试/诊断)。
func (s *Store) CommandState(commandID string) (state, consumer string, err error) {
err = s.db.QueryRow(s.q(`SELECT state, COALESCE(consumer,'') FROM run_commands WHERE command_id=?`), commandID).Scan(&state, &consumer)
if errors.Is(err, sql.ErrNoRows) {
return "", "", ErrNotFound
}
return
}
+121
View File
@@ -0,0 +1,121 @@
package store
import (
"errors"
"testing"
"time"
)
// 协作存储 U5(KTD4):会话执行租约与 epoch fencing。
// 双档(PG 由 CI 真库 job 覆盖咨询锁与跨实例语义;SQLite 验证条件逻辑)。
func TestExecutionClaimAndContention(t *testing.T) {
st := openTestStore(t)
// 首次占用:epoch=1。
e1, err := st.TryClaimExecution("cnv_1", "instA", "run_1", 60)
if err != nil || e1 != 1 {
t.Fatalf("claim A: epoch=%d err=%v", e1, err)
}
// 他实例争抢:租约未到期 → ErrExecutionBusy。
_, err = st.TryClaimExecution("cnv_1", "instB", "run_2", 60)
if !errors.Is(err, ErrExecutionBusy) {
t.Fatalf("B should be busy: %v", err)
}
// 同实例重占(新回合):允许,epoch 递增。
e2, err := st.TryClaimExecution("cnv_1", "instA", "run_3", 60)
if err != nil || e2 != 2 {
t.Fatalf("re-claim A: epoch=%d err=%v", e2, err)
}
// 续租 fencing:正确 (instance,epoch) 成功;旧 epoch 失败。
if !st.RenewExecution("cnv_1", "instA", e2, 60) {
t.Fatal("renew with current epoch should succeed")
}
if st.RenewExecution("cnv_1", "instA", e1, 60) {
t.Fatal("renew with stale epoch must fail")
}
// 主动释放后他实例可占,epoch 继续递增(单调 fencing)。
st.ReleaseExecution("cnv_1", "instA", e2)
e3, err := st.TryClaimExecution("cnv_1", "instB", "run_4", 60)
if err != nil || e3 != 3 {
t.Fatalf("takeover B: epoch=%d err=%v", e3, err)
}
// fence 校验:B 当前 OK;A 的旧代次失败。
if !st.ExecutionFenceOK("cnv_1", "instB", e3) {
t.Fatal("B fence should be ok")
}
if st.ExecutionFenceOK("cnv_1", "instA", e2) {
t.Fatal("A stale fence must fail")
}
}
func TestExecutionLeaseExpiryTakeover(t *testing.T) {
st := openTestStore(t)
// 1 秒租约到期后他实例接管。
if _, err := st.TryClaimExecution("cnv_2", "instA", "run_1", 1); err != nil {
t.Fatal(err)
}
time.Sleep(1100 * time.Millisecond)
e2, err := st.TryClaimExecution("cnv_2", "instB", "run_2", 60)
if err != nil || e2 != 2 {
t.Fatalf("expired takeover: epoch=%d err=%v", e2, err)
}
// 旧 owner 续租被拒(fencing)。
if st.RenewExecution("cnv_2", "instA", 1, 60) {
t.Fatal("stale renew must fail after takeover")
}
}
// 命令收件箱(KTD5):幂等投递、原子认领、并发认领不重复。
func TestRunCommandsClaim(t *testing.T) {
st := openTestStore(t)
if err := st.EnqueueCommand("cmd_1", "cancel", "run_9", "cnv_9", 0, map[string]string{"owner": "o"}); err != nil {
t.Fatal(err)
}
// 幂等:重复投递不新增。
if err := st.EnqueueCommand("cmd_1", "cancel", "run_9", "cnv_9", 0, nil); err != nil {
t.Fatal(err)
}
got, err := st.ClaimPendingCommands("instA", 8)
if err != nil || len(got) != 1 || got[0].CommandID != "cmd_1" || got[0].Kind != "cancel" {
t.Fatalf("claim: %v err=%v", got, err)
}
state, consumer, err := st.CommandState("cmd_1")
if err != nil || state != "consumed" || consumer != "instA" {
t.Fatalf("state=%s consumer=%s err=%v", state, consumer, err)
}
// 已消费不再返回。
got, err = st.ClaimPendingCommands("instB", 8)
if err != nil || len(got) != 0 {
t.Fatalf("consumed must not re-deliver: %v err=%v", got, err)
}
}
func TestRunCommandsConcurrentSingleClaim(t *testing.T) {
st := openTestStore(t)
for i := 0; i < 8; i++ {
if err := st.EnqueueCommand(newID("cmd"), "cancel", newID("run"), "cnv_c", 0, nil); err != nil {
t.Fatal(err)
}
}
var mu = make(chan int, 2)
total := 0
done := make(chan struct{})
for w := 0; w < 2; w++ {
go func(w int) {
got, err := st.ClaimPendingCommands("inst"+string(rune('A'+w)), 8)
if err == nil {
mu <- len(got)
}
done <- struct{}{}
}(w)
}
<-done
<-done
close(mu)
for n := range mu {
total += n
}
if total != 8 {
t.Fatalf("并发认领应恰好 8 条(不重复不丢失):total=%d", total)
}
}
+57
View File
@@ -592,6 +592,63 @@ func init() {
`DROP TABLE IF EXISTS surface_revisions`,
`DROP TABLE IF EXISTS surfaces`,
}}},
migration{version: 23, name: "execution_ownership_commands",
// 执行所有权与持久命令(协作存储 U5,KTD4/KTD5):
// conversation_executions = 会话执行租约(实例/run/epoch/到期),
// 接管递增 epoch 作 fencing;run_commands = 跨实例命令收件箱
//(取消/HITL:command_id 幂等、state pending/consumed)。
up: map[string][]string{"sqlite": {
`CREATE TABLE IF NOT EXISTS conversation_executions (
conversation_id TEXT PRIMARY KEY,
instance_id TEXT NOT NULL,
run_id TEXT NOT NULL DEFAULT '',
epoch INTEGER NOT NULL DEFAULT 0,
lease_until TEXT NOT NULL,
updated_at TEXT NOT NULL
)`,
`CREATE TABLE IF NOT EXISTS run_commands (
command_id TEXT PRIMARY KEY,
kind TEXT NOT NULL,
run_id TEXT NOT NULL DEFAULT '',
conversation_id TEXT NOT NULL DEFAULT '',
epoch INTEGER NOT NULL DEFAULT 0,
payload TEXT NOT NULL DEFAULT '{}',
state TEXT NOT NULL DEFAULT 'pending',
created_at TEXT NOT NULL,
consumed_at TEXT,
consumer TEXT NOT NULL DEFAULT ''
)`,
`CREATE INDEX IF NOT EXISTS idx_run_commands_pending ON run_commands(state, created_at)`,
}, "postgres": {
`CREATE TABLE IF NOT EXISTS conversation_executions (
conversation_id TEXT PRIMARY KEY,
instance_id TEXT NOT NULL,
run_id TEXT NOT NULL DEFAULT '',
epoch INTEGER NOT NULL DEFAULT 0,
lease_until TEXT NOT NULL,
updated_at TEXT NOT NULL
)`,
`CREATE TABLE IF NOT EXISTS run_commands (
command_id TEXT PRIMARY KEY,
kind TEXT NOT NULL,
run_id TEXT NOT NULL DEFAULT '',
conversation_id TEXT NOT NULL DEFAULT '',
epoch INTEGER NOT NULL DEFAULT 0,
payload TEXT NOT NULL DEFAULT '{}',
state TEXT NOT NULL DEFAULT 'pending',
created_at TEXT NOT NULL,
consumed_at TEXT,
consumer TEXT NOT NULL DEFAULT ''
)`,
`CREATE INDEX IF NOT EXISTS idx_run_commands_pending ON run_commands(state, created_at)`,
}},
down: map[string][]string{"sqlite": {
`DROP TABLE IF EXISTS run_commands`,
`DROP TABLE IF EXISTS conversation_executions`,
}, "postgres": {
`DROP TABLE IF EXISTS run_commands`,
`DROP TABLE IF EXISTS conversation_executions`,
}}},
)
}
+65 -11
View File
@@ -122,6 +122,42 @@ func (s *Store) ActiveHITLCardsByRun(runID string) ([]core.HITLCard, error) {
// SettleRun 在一个事务里更新 run,并可选地追加助手消息。
// updated 为假表示条件更新没有改到行(已被收成终态);消息仍会写入。
// 序号冲突时整段事务重试。其它失败回滚,run 保持原状态。
// SettleRunFenced(协作存储 U5/KTD4):终态结算带执行围栏——事务内校验
// (instance,epoch) 仍是该会话的当前执行权,被接管则拒写(fenced=true,run
// 不变)。无租约行(未启用执行所有权)恒放行,语义与 SettleRun 一致。
func (s *Store) SettleRunFenced(r *core.Run, msg *core.Message, instanceID string, epoch int64) (updated, fenced bool, err error) {
s.settleMu.Lock()
fault := s.settleFault
s.settleMu.Unlock()
if fault != nil {
return false, false, fault
}
if msg != nil && msg.ID == "" {
msg.ID = newID("msg")
}
var last error
for attempt := 0; attempt < 8; attempt++ {
updated, fenced, last = s.settleRunOnce(r, msg, &fenceRef{instance: instanceID, epoch: epoch, conversation: string(r.ConversationID)})
if last == nil {
return updated, fenced, nil
}
if !isUniqueViolation(last) {
return false, false, last
}
time.Sleep(time.Duration(attempt) * time.Millisecond)
}
if last == nil {
last = fmt.Errorf("store: settle retries exhausted (run %s)", r.ID)
}
return false, false, last
}
// fenceRef 执行围栏引用(nil = 不校验)。
type fenceRef struct {
instance, conversation string
epoch int64
}
func (s *Store) SettleRun(r *core.Run, msg *core.Message) (updated bool, err error) {
s.settleMu.Lock()
fault := s.settleFault
@@ -134,9 +170,10 @@ func (s *Store) SettleRun(r *core.Run, msg *core.Message) (updated bool, err err
}
var last error
for attempt := 0; attempt < 8; attempt++ {
updated, last = s.settleRunOnce(r, msg)
var fenced bool
updated, fenced, last = s.settleRunOnce(r, msg, nil)
if last == nil {
return updated, nil
return updated && !fenced, nil
}
if !isUniqueViolation(last) {
return false, last
@@ -149,14 +186,31 @@ func (s *Store) SettleRun(r *core.Run, msg *core.Message) (updated bool, err err
return false, last
}
func (s *Store) settleRunOnce(r *core.Run, msg *core.Message) (bool, error) {
func (s *Store) settleRunOnce(r *core.Run, msg *core.Message, fence *fenceRef) (bool, bool, error) {
tx, err := s.db.Begin()
if err != nil {
return false, err
return false, false, err
}
defer tx.Rollback()
if err := s.dialect.lockConversation(tx, string(r.ConversationID)); err != nil {
return false, err
return false, false, err
}
// 执行围栏(KTD4):会话存在租约行且 (instance,epoch) 不匹配 → 拒写。
// 无行 = 该会话未启用执行所有权(本地档/旧路径),放行。
if fence != nil {
var n int
if err := tx.QueryRow(s.q(`SELECT COUNT(*) FROM conversation_executions WHERE conversation_id=?`), fence.conversation).Scan(&n); err != nil {
return false, false, err
}
if n > 0 {
var m int
if err := tx.QueryRow(s.q(`SELECT COUNT(*) FROM conversation_executions WHERE conversation_id=? AND instance_id=? AND epoch=?`), fence.conversation, fence.instance, fence.epoch).Scan(&m); err != nil {
return false, false, err
}
if m == 0 {
return false, true, nil // fenced:旧执行代次,run 不变
}
}
}
var ended any
if r.EndedAt != nil {
@@ -165,17 +219,17 @@ func (s *Store) settleRunOnce(r *core.Run, msg *core.Message) (bool, error) {
res, err := tx.Exec(s.q(`UPDATE runs SET status=?, phase=?, usage=?, error=?, ended_at=? WHERE id=? AND status IN ('pending','running','waiting_hitl')`),
string(r.Status), string(r.Phase), marshalJSON(r.Usage), marshalJSON(r.Error), ended, r.ID)
if err != nil {
return false, err
return false, false, err
}
n, err := res.RowsAffected()
if err != nil {
return false, err
return false, false, err
}
if msg != nil {
insert := `INSERT INTO messages(id,conversation_id,run_id,role,parts,seq,created_at,request_id)
SELECT ?,?,?,?,?,COALESCE(MAX(seq),0)+1,?,? FROM messages WHERE conversation_id=?`
if _, err := tx.Exec(s.q(insert), msg.ID, msg.ConversationID, msg.RunID, string(msg.Role), marshalJSON(msg.Parts), nowISO(), "", msg.ConversationID); err != nil {
return false, err
return false, false, err
}
if msg.VariantGroup != "" || msg.Excluded {
ex := 0
@@ -183,17 +237,17 @@ func (s *Store) settleRunOnce(r *core.Run, msg *core.Message) (bool, error) {
ex = 1
}
if _, err := tx.Exec(s.q(`UPDATE messages SET excluded=?, variant_group=?, variant_index=? WHERE id=?`), ex, msg.VariantGroup, msg.VariantIndex, msg.ID); err != nil {
return false, err
return false, false, err
}
}
}
if err := tx.Commit(); err != nil {
return false, err
return false, false, err
}
if msg != nil {
s.TouchConversation(string(msg.ConversationID))
}
return n == 1, nil
return n == 1, false, nil
}
// ListInterruptedSubagentRuns 列出被收成 interrupted 的子代理 run