Merge pull request 'Add optional harness hooks to the native turn loop' (#6) from feature/agent-harness-p1 into main
This commit was merged in pull request #6.
This commit is contained in:
+1
-1
@@ -164,7 +164,7 @@ func emitParsed(ctx context.Context, r io.Reader, out chan<- Event) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
emitAbort(out, content.String(), ctx.Err())
|
||||
case out <- Event{Kind: KindMessageEnd, Message: msg, Usage: usage}:
|
||||
case out <- Event{Kind: KindMessageEnd, Message: msg, Usage: usage, FinishReason: finished}:
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -38,6 +38,29 @@ func TestParseSSETextAndUsage(t *testing.T) {
|
||||
if end.Usage.TotalTokens != 5 {
|
||||
t.Fatalf("usage %#v", end.Usage)
|
||||
}
|
||||
if end.FinishReason != "stop" {
|
||||
t.Fatalf("finish reason %q, want stop", end.FinishReason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSSEFinishReasonLengthKeepsStopEnd(t *testing.T) {
|
||||
body := strings.Join([]string{
|
||||
`data: {"choices":[{"delta":{"content":"半"},"finish_reason":"length"}]}`,
|
||||
`data: [DONE]`,
|
||||
"",
|
||||
}, "\n")
|
||||
var end Event
|
||||
for ev := range ParseSSE(context.Background(), strings.NewReader(body)) {
|
||||
if ev.Kind == KindMessageEnd {
|
||||
end = ev
|
||||
}
|
||||
}
|
||||
if end.FinishReason != "length" {
|
||||
t.Fatalf("finish reason %q, want length", end.FinishReason)
|
||||
}
|
||||
if end.Message.StopReason != StopEnd {
|
||||
t.Fatalf("stop reason %s, want end so existing callers stay unchanged", end.Message.StopReason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSSEToolCallIncrements(t *testing.T) {
|
||||
|
||||
@@ -90,9 +90,12 @@ type Event struct {
|
||||
ArgumentsDelta string
|
||||
Message Message
|
||||
Usage Usage
|
||||
Err error
|
||||
Retryable bool
|
||||
ErrorMessage string
|
||||
// FinishReason 是供应商原始结束原因(stop、length、tool_calls 等)。
|
||||
// 空表示这条事件不携带结束原因。StopReason 的映射不受此字段影响。
|
||||
FinishReason string
|
||||
Err error
|
||||
Retryable bool
|
||||
ErrorMessage string
|
||||
}
|
||||
|
||||
// StreamOptions tunes one provider request.
|
||||
|
||||
+192
-11
@@ -18,8 +18,15 @@ import (
|
||||
const (
|
||||
ExecSequential = "sequential"
|
||||
ExecParallel = "parallel"
|
||||
|
||||
maxStopContinuations = 3
|
||||
maxLengthContinuations = 3
|
||||
maxRepeatReminders = 3
|
||||
)
|
||||
|
||||
const repeatReminderText = "你已连续 3 次以相同参数调用同一工具。如果结果不符合预期,请更换参数或做法,不要再重复这一调用。"
|
||||
const lengthContinuePrompt = "上一段回复因为达到长度上限被截断。请从断点接着写,不要重复已经写出的内容。"
|
||||
|
||||
// Event is a lifecycle notice for logs and UI.
|
||||
type Event struct {
|
||||
Type string
|
||||
@@ -29,9 +36,11 @@ type Event struct {
|
||||
// Detail carries tool arguments/results or reasoning deltas for
|
||||
// process-visualization consumers; log writers may ignore it.
|
||||
Detail string
|
||||
// PromptTokens / CompletionTokens 只在 type=usage 时有意义。
|
||||
// PromptTokens / CompletionTokens / TotalTokens 在 type=usage 或
|
||||
// type=message_end 时有意义,供调用方累计用量。
|
||||
PromptTokens int
|
||||
CompletionTokens int
|
||||
TotalTokens int
|
||||
}
|
||||
|
||||
// Config drives one agent session.
|
||||
@@ -57,6 +66,41 @@ type Config struct {
|
||||
// Budget 可选。任一上限命中即停止并返回 budget_exceeded 错误(StopError);
|
||||
// 零值字段不设限。截止时刻与外层 ctx 先到者生效。
|
||||
Budget *Budget
|
||||
// History 是会话转录为空时的初始转录。以 system 开头时,不再前置
|
||||
// SystemPrompt 和工具清单。不回写进 Session。
|
||||
History []llm.Message
|
||||
// Stop 在模型不再调用工具、且本轮没有 follow-up 时询问是否续跑。
|
||||
// nil 表示不询问。内核最多接受 3 次 Continue。
|
||||
Stop func(last llm.Message, info StopInfo) StopDecision
|
||||
// BeforeTool 返回错误时不执行工具,但仍写一条 IsError 的 tool 结果。
|
||||
BeforeTool func(call llm.ToolCall) error
|
||||
// UnknownTool 处理注册表里没有的调用。nil 时仍返回 unknown tool。
|
||||
UnknownTool func(call llm.ToolCall) tools.Result
|
||||
// AfterTool 只观察工具结果,不能改写。
|
||||
AfterTool func(call llm.ToolCall, res llm.Message)
|
||||
// Reminders 在每次调用模型前提供提醒。正文包在 system-reminder 里,
|
||||
// 只进入本次请求,不写进 Session。
|
||||
Reminders func(turn int) []string
|
||||
// RepeatGuard 为真时,同名且参数按键排序后相同的调用连续 3 次,注入一条提醒。
|
||||
// 每个 run 最多 3 条,不中断循环。
|
||||
RepeatGuard bool
|
||||
// ContinueOnLength 为真时,FinishReason=length 且没有工具调用就续写,最多 3 次。
|
||||
// 续写提示不写进 Session。第 4 次截断返回错误。
|
||||
ContinueOnLength bool
|
||||
}
|
||||
|
||||
// StopInfo 是 Stop 钩子看到的本 run 进度。
|
||||
type StopInfo struct {
|
||||
Continuations int
|
||||
ToolCalls int
|
||||
}
|
||||
|
||||
// StopDecision 决定模型停下之后是否再跑一轮。
|
||||
type StopDecision struct {
|
||||
Continue bool
|
||||
Replacement string
|
||||
Reason string
|
||||
Notice string
|
||||
}
|
||||
|
||||
// Budget 是循环层的执行预算;与 exec.Budget 字段同构,由适配层换算。
|
||||
@@ -73,6 +117,7 @@ type Result struct {
|
||||
Messages []llm.Message
|
||||
StopReason llm.StopReason
|
||||
Usage llm.Usage
|
||||
Notice string
|
||||
Err error
|
||||
Retryable bool
|
||||
}
|
||||
@@ -96,6 +141,9 @@ func Run(ctx context.Context, cfg Config, user llm.Message) (Result, error) {
|
||||
}
|
||||
messages = loaded
|
||||
}
|
||||
if len(messages) == 0 && len(cfg.History) > 0 {
|
||||
messages = append([]llm.Message{}, cfg.History...)
|
||||
}
|
||||
if len(messages) == 0 || messages[0].Role != llm.RoleSystem {
|
||||
sys := llm.Message{Role: llm.RoleSystem, Content: systemText(cfg)}
|
||||
messages = append([]llm.Message{sys}, messages...)
|
||||
@@ -112,6 +160,10 @@ func Run(ctx context.Context, cfg Config, user llm.Message) (Result, error) {
|
||||
var last llm.Message
|
||||
var usage llm.Usage
|
||||
toolCalls := 0
|
||||
stopContinues := 0
|
||||
lengthContinues := 0
|
||||
lengthNudge := false
|
||||
var repeats repeatState
|
||||
for turn := 0; turn < cfg.MaxTurns; turn++ {
|
||||
if ctx.Err() != nil {
|
||||
cfg.logger().Error("runtime turn aborted", ctx.Err())
|
||||
@@ -122,7 +174,8 @@ func Run(ctx context.Context, cfg Config, user llm.Message) (Result, error) {
|
||||
}
|
||||
emit(cfg, Event{Type: "turn_start"})
|
||||
emit(cfg, Event{Type: "message_start"})
|
||||
end, errEv := consume(ctx, cfg, messages)
|
||||
request := withEphemeral(messages, turnReminders(cfg, turn, &repeats, &lengthNudge))
|
||||
end, errEv := consume(ctx, cfg, request)
|
||||
if errEv != nil {
|
||||
cfg.logger().Error("runtime turn failed", errEv, "retryable", end.Retryable)
|
||||
return Result{Text: end.Message.Content, Messages: messages, StopReason: end.Message.StopReason, Err: errEv, Retryable: end.Retryable}, nil
|
||||
@@ -135,7 +188,7 @@ func Run(ctx context.Context, cfg Config, user llm.Message) (Result, error) {
|
||||
return Result{}, err
|
||||
}
|
||||
messages = append(messages, last)
|
||||
emit(cfg, Event{Type: "message_end", Message: last})
|
||||
emit(cfg, Event{Type: "message_end", Message: last, PromptTokens: end.Usage.PromptTokens, CompletionTokens: end.Usage.CompletionTokens, TotalTokens: end.Usage.TotalTokens})
|
||||
|
||||
if last.StopReason == llm.StopToolCalls && len(last.ToolCalls) > 0 {
|
||||
if cfg.Budget != nil && cfg.Budget.MaxToolCalls > 0 && toolCalls+len(last.ToolCalls) > cfg.Budget.MaxToolCalls {
|
||||
@@ -143,6 +196,9 @@ func Run(ctx context.Context, cfg Config, user llm.Message) (Result, error) {
|
||||
return budgetStop(cfg, last, messages, usage, reason), nil
|
||||
}
|
||||
toolCalls += len(last.ToolCalls)
|
||||
if cfg.RepeatGuard {
|
||||
repeats.observe(last.ToolCalls)
|
||||
}
|
||||
results := executeTools(ctx, cfg, last)
|
||||
for _, res := range results {
|
||||
if err := persist(cfg, res); err != nil {
|
||||
@@ -162,6 +218,16 @@ func Run(ctx context.Context, cfg Config, user llm.Message) (Result, error) {
|
||||
continue
|
||||
}
|
||||
emit(cfg, Event{Type: "turn_end", Message: last})
|
||||
if cfg.ContinueOnLength && end.FinishReason == "length" && len(last.ToolCalls) == 0 {
|
||||
if lengthContinues >= maxLengthContinuations {
|
||||
errCap := errString("length continuation exceeded")
|
||||
cfg.logger().Error("runtime length continuation exceeded", errCap)
|
||||
return Result{Text: last.Content, Messages: messages, StopReason: llm.StopError, Usage: usage, Err: errCap}, nil
|
||||
}
|
||||
lengthContinues++
|
||||
lengthNudge = true
|
||||
continue
|
||||
}
|
||||
if follow := drain(cfg.GetFollowUp); len(follow) > 0 {
|
||||
for _, m := range follow {
|
||||
if err := persist(cfg, m); err != nil {
|
||||
@@ -171,10 +237,26 @@ func Run(ctx context.Context, cfg Config, user llm.Message) (Result, error) {
|
||||
}
|
||||
continue
|
||||
}
|
||||
emit(cfg, Event{Type: "agent_end", Message: last})
|
||||
emit(cfg, Event{Type: "agent_settled", Message: last})
|
||||
emitUsage(cfg, usage)
|
||||
return Result{Text: last.Content, Messages: messages, StopReason: last.StopReason, Usage: usage}, nil
|
||||
if cfg.Stop != nil {
|
||||
decision := cfg.Stop(last, StopInfo{Continuations: stopContinues, ToolCalls: toolCalls})
|
||||
if decision.Continue && stopContinues < maxStopContinuations {
|
||||
stopContinues++
|
||||
if decision.Replacement != "" && len(messages) > 0 {
|
||||
messages[len(messages)-1].Content = decision.Replacement
|
||||
last.Content = decision.Replacement
|
||||
}
|
||||
if decision.Reason != "" {
|
||||
reasonMsg := llm.Message{Role: llm.RoleUser, Content: decision.Reason}
|
||||
if err := persist(cfg, reasonMsg); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
messages = append(messages, reasonMsg)
|
||||
}
|
||||
continue
|
||||
}
|
||||
return settle(cfg, last, messages, usage, decision.Notice), nil
|
||||
}
|
||||
return settle(cfg, last, messages, usage, ""), nil
|
||||
}
|
||||
errCap := errString("max turns exceeded")
|
||||
cfg.logger().Error("runtime turn cap hit", errCap)
|
||||
@@ -290,19 +372,118 @@ func executeParallel(ctx context.Context, cfg Config, assistant llm.Message) []l
|
||||
|
||||
func runOne(ctx context.Context, cfg Config, call llm.ToolCall) llm.Message {
|
||||
emit(cfg, Event{Type: "tool_execution_start", Tool: call.Name, Detail: call.Arguments})
|
||||
if cfg.BeforeTool != nil {
|
||||
if err := cfg.BeforeTool(call); err != nil {
|
||||
msg := llm.Message{Role: llm.RoleTool, Name: call.Name, ToolCallID: call.ID, Content: err.Error(), IsError: true}
|
||||
emit(cfg, Event{Type: "tool_execution_end", Tool: call.Name, Text: msg.Content})
|
||||
if cfg.AfterTool != nil {
|
||||
cfg.AfterTool(call, msg)
|
||||
}
|
||||
return msg
|
||||
}
|
||||
}
|
||||
emit(cfg, Event{Type: "tool_execution_update", Tool: call.Name, Detail: call.Arguments})
|
||||
res := tools.Result{Content: "no tools configured", IsError: true}
|
||||
if cfg.Tools != nil {
|
||||
known := cfg.Tools != nil && cfg.Tools.Has(call.Name)
|
||||
switch {
|
||||
case known:
|
||||
res = cfg.Tools.Execute(ctx, call.Name, json.RawMessage(call.Arguments))
|
||||
case cfg.UnknownTool != nil:
|
||||
res = cfg.UnknownTool(call)
|
||||
case cfg.Tools != nil:
|
||||
res = cfg.Tools.Execute(ctx, call.Name, json.RawMessage(call.Arguments))
|
||||
}
|
||||
emit(cfg, Event{Type: "tool_execution_end", Tool: call.Name, Text: res.Content})
|
||||
return llm.Message{
|
||||
msg := llm.Message{
|
||||
Role: llm.RoleTool,
|
||||
Name: call.Name,
|
||||
ToolCallID: call.ID,
|
||||
Content: res.Content,
|
||||
IsError: res.IsError,
|
||||
}
|
||||
emit(cfg, Event{Type: "tool_execution_end", Tool: call.Name, Text: res.Content})
|
||||
if cfg.AfterTool != nil {
|
||||
cfg.AfterTool(call, msg)
|
||||
}
|
||||
return msg
|
||||
}
|
||||
|
||||
func settle(cfg Config, last llm.Message, messages []llm.Message, usage llm.Usage, notice string) Result {
|
||||
emit(cfg, Event{Type: "agent_end", Message: last})
|
||||
emit(cfg, Event{Type: "agent_settled", Message: last})
|
||||
emitUsage(cfg, usage)
|
||||
return Result{Text: last.Content, Messages: messages, StopReason: last.StopReason, Usage: usage, Notice: notice}
|
||||
}
|
||||
|
||||
func reminderMessage(text string) llm.Message {
|
||||
return llm.Message{Role: llm.RoleUser, Content: "<system-reminder>\n" + text + "\n</system-reminder>"}
|
||||
}
|
||||
|
||||
func turnReminders(cfg Config, turn int, repeats *repeatState, lengthNudge *bool) []llm.Message {
|
||||
var extra []llm.Message
|
||||
if cfg.Reminders != nil {
|
||||
for _, text := range cfg.Reminders(turn) {
|
||||
if strings.TrimSpace(text) == "" {
|
||||
continue
|
||||
}
|
||||
extra = append(extra, reminderMessage(text))
|
||||
}
|
||||
}
|
||||
if lengthNudge != nil && *lengthNudge {
|
||||
extra = append(extra, llm.Message{Role: llm.RoleUser, Content: lengthContinuePrompt})
|
||||
*lengthNudge = false
|
||||
}
|
||||
if cfg.RepeatGuard && repeats != nil {
|
||||
due := repeats.streak/3 - repeats.reminders
|
||||
room := maxRepeatReminders - repeats.reminders
|
||||
if due > room {
|
||||
due = room
|
||||
}
|
||||
for i := 0; i < due; i++ {
|
||||
extra = append(extra, reminderMessage(repeatReminderText))
|
||||
repeats.reminders++
|
||||
}
|
||||
}
|
||||
return extra
|
||||
}
|
||||
|
||||
func withEphemeral(messages, extra []llm.Message) []llm.Message {
|
||||
if len(extra) == 0 {
|
||||
return messages
|
||||
}
|
||||
out := make([]llm.Message, 0, len(messages)+len(extra))
|
||||
out = append(out, messages...)
|
||||
out = append(out, extra...)
|
||||
return out
|
||||
}
|
||||
|
||||
type repeatState struct {
|
||||
sig string
|
||||
streak int
|
||||
reminders int
|
||||
}
|
||||
|
||||
func (s *repeatState) observe(calls []llm.ToolCall) {
|
||||
for _, call := range calls {
|
||||
sig := call.Name + "\n" + canonicalArgs(call.Arguments)
|
||||
if sig == s.sig {
|
||||
s.streak++
|
||||
continue
|
||||
}
|
||||
s.sig = sig
|
||||
s.streak = 1
|
||||
}
|
||||
}
|
||||
|
||||
func canonicalArgs(raw string) string {
|
||||
var v any
|
||||
if err := json.Unmarshal([]byte(raw), &v); err != nil {
|
||||
return raw
|
||||
}
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return raw
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func persist(cfg Config, msg llm.Message) error {
|
||||
@@ -320,7 +501,7 @@ func drain(fn func() []llm.Message) []llm.Message {
|
||||
}
|
||||
|
||||
func emitUsage(cfg Config, usage llm.Usage) {
|
||||
emit(cfg, Event{Type: "usage", PromptTokens: usage.PromptTokens, CompletionTokens: usage.CompletionTokens})
|
||||
emit(cfg, Event{Type: "usage", PromptTokens: usage.PromptTokens, CompletionTokens: usage.CompletionTokens, TotalTokens: usage.TotalTokens})
|
||||
}
|
||||
|
||||
func emit(cfg Config, ev Event) {
|
||||
|
||||
@@ -0,0 +1,332 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"agent-native/runtime/llm"
|
||||
"agent-native/runtime/session"
|
||||
"agent-native/runtime/tools"
|
||||
)
|
||||
|
||||
func TestLoopStopContinuesUntilCap(t *testing.T) {
|
||||
var calls int
|
||||
res, err := Run(context.Background(), Config{
|
||||
Model: "fake",
|
||||
MaxTurns: 8,
|
||||
Stream: script(
|
||||
llm.Message{Role: llm.RoleAssistant, Content: "forged-claim", StopReason: llm.StopEnd},
|
||||
llm.Message{Role: llm.RoleAssistant, Content: "forged-claim", StopReason: llm.StopEnd},
|
||||
llm.Message{Role: llm.RoleAssistant, Content: "forged-claim", StopReason: llm.StopEnd},
|
||||
llm.Message{Role: llm.RoleAssistant, Content: "forged-claim", StopReason: llm.StopEnd},
|
||||
),
|
||||
Stop: func(last llm.Message, info StopInfo) StopDecision {
|
||||
calls++
|
||||
if info.Continuations != calls-1 {
|
||||
t.Errorf("continuations = %d, want %d", info.Continuations, calls-1)
|
||||
}
|
||||
return StopDecision{Continue: true, Replacement: "已作废", Reason: "再试", Notice: "到顶了"}
|
||||
},
|
||||
}, llm.Message{Role: llm.RoleUser, Content: "go"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls != 4 {
|
||||
t.Fatalf("stop calls = %d, want 4", calls)
|
||||
}
|
||||
if res.Err != nil || res.Notice != "到顶了" {
|
||||
t.Fatalf("notice %q err %v", res.Notice, res.Err)
|
||||
}
|
||||
replaced, forged := 0, 0
|
||||
for _, m := range res.Messages {
|
||||
if m.Role != llm.RoleAssistant {
|
||||
continue
|
||||
}
|
||||
if m.Content == "已作废" {
|
||||
replaced++
|
||||
}
|
||||
if m.Content == "forged-claim" {
|
||||
forged++
|
||||
}
|
||||
}
|
||||
if replaced != 3 || forged != 1 {
|
||||
t.Fatalf("replaced=%d forged=%d transcript=%s", replaced, forged, transcriptText(res.Messages))
|
||||
}
|
||||
if strings.Count(transcriptText(res.Messages), "再试") != 3 {
|
||||
t.Fatalf("reasons = %s", transcriptText(res.Messages))
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoopBeforeToolRejectPairsResult(t *testing.T) {
|
||||
hits := 0
|
||||
reg := tools.NewRegistry(t.TempDir())
|
||||
reg.Register(hitTool{hits: &hits})
|
||||
var seen llm.Message
|
||||
res, err := Run(context.Background(), Config{
|
||||
Model: "fake",
|
||||
Tools: reg,
|
||||
Stream: script(
|
||||
llm.Message{Role: llm.RoleAssistant, StopReason: llm.StopToolCalls, ToolCalls: []llm.ToolCall{{
|
||||
ID: "c1", Name: "echo", Arguments: `{}`,
|
||||
}}},
|
||||
llm.Message{Role: llm.RoleAssistant, Content: "done", StopReason: llm.StopEnd},
|
||||
),
|
||||
BeforeTool: func(call llm.ToolCall) error {
|
||||
return errors.New("不允许")
|
||||
},
|
||||
AfterTool: func(call llm.ToolCall, msg llm.Message) {
|
||||
seen = msg
|
||||
},
|
||||
}, llm.Message{Role: llm.RoleUser, Content: "go"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if hits != 0 {
|
||||
t.Fatalf("tool executed %d times", hits)
|
||||
}
|
||||
if !seen.IsError || seen.Content != "不允许" || seen.ToolCallID != "c1" {
|
||||
t.Fatalf("after = %#v", seen)
|
||||
}
|
||||
paired := false
|
||||
for _, m := range res.Messages {
|
||||
if m.Role == llm.RoleTool && m.ToolCallID == "c1" && m.IsError && m.Content == "不允许" {
|
||||
paired = true
|
||||
}
|
||||
}
|
||||
if !paired {
|
||||
t.Fatalf("missing paired tool result: %s", transcriptText(res.Messages))
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoopRemindersStayOutOfSession(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "transcript.jsonl")
|
||||
st, err := session.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var saw bool
|
||||
_, err = Run(context.Background(), Config{
|
||||
Model: "fake",
|
||||
Session: st,
|
||||
Stream: func(ctx context.Context, model string, messages []llm.Message, decls []llm.ToolDecl, opts llm.StreamOptions) <-chan llm.Event {
|
||||
for _, m := range messages {
|
||||
if strings.Contains(m.Content, "<system-reminder>") && strings.Contains(m.Content, "先核对交付物") {
|
||||
saw = true
|
||||
}
|
||||
}
|
||||
ch := make(chan llm.Event, 1)
|
||||
ch <- llm.Event{Kind: llm.KindMessageEnd, Message: llm.Message{Role: llm.RoleAssistant, Content: "ok", StopReason: llm.StopEnd}}
|
||||
close(ch)
|
||||
return ch
|
||||
},
|
||||
Reminders: func(turn int) []string {
|
||||
if turn != 0 {
|
||||
t.Errorf("turn %d", turn)
|
||||
}
|
||||
return []string{"先核对交付物"}
|
||||
},
|
||||
}, llm.Message{Role: llm.RoleUser, Content: "ping"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !saw {
|
||||
t.Fatal("reminder missing from the provider request")
|
||||
}
|
||||
loaded, err := st.Load()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(transcriptText(loaded), "先核对交付物") {
|
||||
t.Fatalf("reminder persisted: %#v", loaded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoopRepeatGuardFiresAndCaps(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
var requests []string
|
||||
turns := make([]llm.Message, 0, 13)
|
||||
for i := 0; i < 12; i++ {
|
||||
args := `{"b":1,"a":2}`
|
||||
if i%2 == 1 {
|
||||
args = `{"a":2,"b":1}`
|
||||
}
|
||||
turns = append(turns, llm.Message{Role: llm.RoleAssistant, StopReason: llm.StopToolCalls, ToolCalls: []llm.ToolCall{{
|
||||
ID: "c", Name: "echo", Arguments: args,
|
||||
}}})
|
||||
}
|
||||
turns = append(turns, llm.Message{Role: llm.RoleAssistant, Content: "done", StopReason: llm.StopEnd})
|
||||
var i int
|
||||
stream := func(ctx context.Context, model string, messages []llm.Message, decls []llm.ToolDecl, opts llm.StreamOptions) <-chan llm.Event {
|
||||
mu.Lock()
|
||||
requests = append(requests, transcriptText(messages))
|
||||
n := i
|
||||
i++
|
||||
mu.Unlock()
|
||||
ch := make(chan llm.Event, 1)
|
||||
msg := turns[len(turns)-1]
|
||||
if n < len(turns) {
|
||||
msg = turns[n]
|
||||
}
|
||||
ch <- llm.Event{Kind: llm.KindMessageEnd, Message: msg}
|
||||
close(ch)
|
||||
return ch
|
||||
}
|
||||
res, err := Run(context.Background(), Config{
|
||||
Model: "fake",
|
||||
MaxTurns: 20,
|
||||
RepeatGuard: true,
|
||||
Stream: stream,
|
||||
}, llm.Message{Role: llm.RoleUser, Content: "go"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Text != "done" {
|
||||
t.Fatalf("text %q", res.Text)
|
||||
}
|
||||
reminders := 0
|
||||
for _, req := range requests {
|
||||
reminders += strings.Count(req, repeatReminderText)
|
||||
}
|
||||
if reminders != 3 {
|
||||
t.Fatalf("reminders = %d, want 3; requests=%d", reminders, len(requests))
|
||||
}
|
||||
if strings.Contains(transcriptText(res.Messages), repeatReminderText) {
|
||||
t.Fatal("repeat reminder was written into the transcript")
|
||||
}
|
||||
if !strings.Contains(requests[3], repeatReminderText) {
|
||||
t.Fatal("first reminder should arrive on the request after the third identical call")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoopContinueOnLengthCaps(t *testing.T) {
|
||||
var requests []string
|
||||
var n int
|
||||
stream := func(ctx context.Context, model string, messages []llm.Message, decls []llm.ToolDecl, opts llm.StreamOptions) <-chan llm.Event {
|
||||
requests = append(requests, transcriptText(messages))
|
||||
n++
|
||||
ch := make(chan llm.Event, 1)
|
||||
ch <- llm.Event{
|
||||
Kind: llm.KindMessageEnd,
|
||||
FinishReason: "length",
|
||||
Message: llm.Message{Role: llm.RoleAssistant, Content: "part", StopReason: llm.StopEnd},
|
||||
}
|
||||
close(ch)
|
||||
return ch
|
||||
}
|
||||
res, err := Run(context.Background(), Config{
|
||||
Model: "fake",
|
||||
MaxTurns: 8,
|
||||
ContinueOnLength: true,
|
||||
Stream: stream,
|
||||
}, llm.Message{Role: llm.RoleUser, Content: "go"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Err == nil || !strings.Contains(res.Err.Error(), "length continuation exceeded") {
|
||||
t.Fatalf("err = %v", res.Err)
|
||||
}
|
||||
if n != 4 {
|
||||
t.Fatalf("model calls = %d, want 4", n)
|
||||
}
|
||||
if len(res.Messages) == 0 || !strings.Contains(transcriptText(res.Messages), "part") {
|
||||
t.Fatal("truncated transcript was dropped")
|
||||
}
|
||||
if strings.Contains(transcriptText(res.Messages), lengthContinuePrompt) {
|
||||
t.Fatal("continuation prompt was persisted")
|
||||
}
|
||||
prompts := 0
|
||||
for _, req := range requests {
|
||||
prompts += strings.Count(req, lengthContinuePrompt)
|
||||
}
|
||||
if prompts != 3 {
|
||||
t.Fatalf("continuation prompts = %d, want 3", prompts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoopHistorySkipsGeneratedSystem(t *testing.T) {
|
||||
var first string
|
||||
_, err := Run(context.Background(), Config{
|
||||
Model: "fake",
|
||||
SystemPrompt: "LEAK",
|
||||
History: []llm.Message{
|
||||
{Role: llm.RoleSystem, Content: "FROM_HISTORY"},
|
||||
{Role: llm.RoleUser, Content: "earlier"},
|
||||
},
|
||||
Stream: func(ctx context.Context, model string, messages []llm.Message, decls []llm.ToolDecl, opts llm.StreamOptions) <-chan llm.Event {
|
||||
if len(messages) > 0 {
|
||||
first = messages[0].Content
|
||||
}
|
||||
ch := make(chan llm.Event, 1)
|
||||
ch <- llm.Event{Kind: llm.KindMessageEnd, Message: llm.Message{Role: llm.RoleAssistant, Content: "ok", StopReason: llm.StopEnd}}
|
||||
close(ch)
|
||||
return ch
|
||||
},
|
||||
}, llm.Message{Role: llm.RoleUser, Content: "now"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first != "FROM_HISTORY" || strings.Contains(first, "LEAK") {
|
||||
t.Fatalf("system = %q", first)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoopUnknownToolFallsThrough(t *testing.T) {
|
||||
res, err := Run(context.Background(), Config{
|
||||
Model: "fake",
|
||||
UnknownTool: func(call llm.ToolCall) tools.Result {
|
||||
return tools.Result{Content: "via " + call.Name}
|
||||
},
|
||||
Stream: script(
|
||||
llm.Message{Role: llm.RoleAssistant, StopReason: llm.StopToolCalls, ToolCalls: []llm.ToolCall{{
|
||||
ID: "c1", Name: "make_report", Arguments: `{}`,
|
||||
}}},
|
||||
llm.Message{Role: llm.RoleAssistant, Content: "done", StopReason: llm.StopEnd},
|
||||
),
|
||||
}, llm.Message{Role: llm.RoleUser, Content: "go"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found := false
|
||||
for _, m := range res.Messages {
|
||||
if m.Role == llm.RoleTool && m.Content == "via make_report" && !m.IsError {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("unknown tool was not delegated: %s", transcriptText(res.Messages))
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoopTurnCapKeepsTranscript(t *testing.T) {
|
||||
res, err := Run(context.Background(), Config{
|
||||
Model: "fake",
|
||||
MaxTurns: 1,
|
||||
Stream: script(llm.Message{Role: llm.RoleAssistant, StopReason: llm.StopToolCalls, ToolCalls: []llm.ToolCall{{
|
||||
ID: "c1", Name: "missing", Arguments: `{}`,
|
||||
}}}),
|
||||
}, llm.Message{Role: llm.RoleUser, Content: "ping"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Err == nil || !strings.Contains(res.Err.Error(), "max turns exceeded") {
|
||||
t.Fatalf("err = %v", res.Err)
|
||||
}
|
||||
if !strings.Contains(transcriptText(res.Messages), "ping") {
|
||||
t.Fatalf("transcript dropped: %s", transcriptText(res.Messages))
|
||||
}
|
||||
}
|
||||
|
||||
type hitTool struct{ hits *int }
|
||||
|
||||
func (h hitTool) Name() string { return "echo" }
|
||||
func (h hitTool) Description() string { return "echo" }
|
||||
func (h hitTool) Parameters() json.RawMessage { return json.RawMessage(`{"type":"object"}`) }
|
||||
func (h hitTool) Execute(context.Context, json.RawMessage) (tools.Result, error) {
|
||||
*h.hits++
|
||||
return tools.Result{Content: "ran"}, nil
|
||||
}
|
||||
@@ -8,6 +8,30 @@ import (
|
||||
"agent-native/runtime/llm"
|
||||
)
|
||||
|
||||
func TestAgentSessionPromptForwardsStop(t *testing.T) {
|
||||
called := false
|
||||
sess := NewAgentSession(Config{
|
||||
Model: "m",
|
||||
Stream: func(ctx context.Context, model string, messages []llm.Message, td []llm.ToolDecl, opts llm.StreamOptions) <-chan llm.Event {
|
||||
ch := make(chan llm.Event, 1)
|
||||
ch <- llm.Event{Kind: llm.KindMessageEnd, Message: llm.Message{Role: llm.RoleAssistant, Content: "hi", StopReason: llm.StopEnd}}
|
||||
close(ch)
|
||||
return ch
|
||||
},
|
||||
Stop: func(last llm.Message, info StopInfo) StopDecision {
|
||||
called = true
|
||||
return StopDecision{}
|
||||
},
|
||||
})
|
||||
res, err := sess.Prompt(context.Background(), llm.Message{Role: llm.RoleUser, Content: "go"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !called || res.Text != "hi" {
|
||||
t.Fatalf("called=%v text=%q", called, res.Text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentSessionSteerFollowAndAbort(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
step := 0
|
||||
|
||||
Reference in New Issue
Block a user