Merge pull request 'Send current-turn images and keep vision fallbacks capable' (#203) from feature/four-model-vision into main
ci / vet-and-test (push) Successful in 1m2s
ci / vet-and-test (push) Successful in 1m2s
Why: land current-turn image routing on backend main.
Evidence: vet-and-test success on 629c2502ad02e33708fe7ee39afd4b45d1b25483; parent pin 6532fad remains an ancestor.
Impact: vision routing is on backend main.
Refs: #203
This commit was merged in pull request #203.
This commit is contained in:
@@ -4,12 +4,14 @@ package ctxasm
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
agentllm "agent-native/runtime/llm"
|
||||
"agent-system-backend/internal/vision"
|
||||
core "agent-system-core"
|
||||
)
|
||||
|
||||
@@ -134,6 +136,7 @@ func AssembleInput(prof core.AgentProfile, eff core.EffectiveSettings, history [
|
||||
// 此处与整段组装共用同一实现,保证两条路径的当前轮语义一致。
|
||||
func BuildCurrentUser(eff core.EffectiveSettings, currentUser core.Message, attachments []core.ParsedDocument, knowledge []core.RetrievalHit, rep *Report) agentllm.Message {
|
||||
var sb strings.Builder
|
||||
var images []agentllm.Image
|
||||
for _, p := range currentUser.Parts {
|
||||
if p.Type == core.PartText {
|
||||
sb.WriteString(p.Text)
|
||||
@@ -146,6 +149,22 @@ func BuildCurrentUser(eff core.EffectiveSettings, currentUser core.Message, atta
|
||||
}
|
||||
sb.WriteString("\n\n[附件](用户上传文档的解析全文,按原文顺序保留标题/列表/表格;超长时文末标注截断。请依据文档内容完成任务,输出完整不省略)\n")
|
||||
for _, doc := range attachments {
|
||||
if pics := vision.ImagesFromDoc(doc); len(pics) > 0 {
|
||||
kept, notes := vision.Admit(images, pics)
|
||||
images = append(images, kept...)
|
||||
if len(kept) > 0 {
|
||||
fmt.Fprintf(&sb, "### %s\n[图片 %s,图像已随本消息发送]\n", doc.Meta.Filename, doc.Meta.Filename)
|
||||
} else {
|
||||
fmt.Fprintf(&sb, "### %s\n[图片 %s,本轮未发送图像]\n", doc.Meta.Filename, doc.Meta.Filename)
|
||||
}
|
||||
for _, note := range notes {
|
||||
fmt.Fprintf(&sb, "%s\n", note)
|
||||
if rep != nil {
|
||||
rep.Truncations = append(rep.Truncations, fmt.Sprintf("附件 %s:%s", doc.Meta.Filename, note))
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
text := doc.PlainText()
|
||||
if runes := []rune(text); len(runes) > limit {
|
||||
text = string(runes[:limit]) + "…(附件超长截断)"
|
||||
@@ -168,7 +187,7 @@ func BuildCurrentUser(eff core.EffectiveSettings, currentUser core.Message, atta
|
||||
rep.Items = append(rep.Items, fmt.Sprintf("knowledge(%d)", len(knowledge)))
|
||||
}
|
||||
}
|
||||
return agentllm.Message{Role: agentllm.RoleUser, Content: sb.String()}
|
||||
return agentllm.Message{Role: agentllm.RoleUser, Content: sb.String(), Images: images}
|
||||
}
|
||||
|
||||
// maxToolArgRunes 是回放进模型的单次工具参数上限。超限时截断字符串值,保持合法 JSON。
|
||||
@@ -435,6 +454,10 @@ func renderArtifactRef(a *core.ArtifactRef, resolve ArtifactResolver, limit int,
|
||||
if a == nil {
|
||||
return ""
|
||||
}
|
||||
name := a.Name
|
||||
if isImageArtifact(a.Kind, name) {
|
||||
return fmt.Sprintf("此前上传的图片:%s。\n", name)
|
||||
}
|
||||
if resolve == nil {
|
||||
return fmt.Sprintf("此前已生成文件:%s。\n", a.Name)
|
||||
}
|
||||
@@ -454,6 +477,18 @@ func renderArtifactRef(a *core.ArtifactRef, resolve ArtifactResolver, limit int,
|
||||
return fmt.Sprintf("此前已生成文件 %s,内容如下:\n%s\n", art.Name, text)
|
||||
}
|
||||
|
||||
func isImageArtifact(kind, name string) bool {
|
||||
if kind == "image" {
|
||||
return true
|
||||
}
|
||||
switch strings.ToLower(filepath.Ext(name)) {
|
||||
case ".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".svg":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// extractiveSummary 把窗口外历史压成短摘要(不调用模型)。summarizeBeyond 打开时注入。
|
||||
func extractiveSummary(dropped []core.Message) string {
|
||||
var lines []string
|
||||
|
||||
@@ -203,3 +203,47 @@ func contains(items []string, want string) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func TestCurrentImageAttachmentAndHistoryPlaceholder(t *testing.T) {
|
||||
png := []byte{
|
||||
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52,
|
||||
0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x02, 0x00, 0x00, 0x00, 0x90, 0x77, 0x53,
|
||||
0xde, 0x00, 0x00, 0x00, 0x0c, 0x49, 0x44, 0x41, 0x54, 0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0x50,
|
||||
0x0f, 0x00, 0x04, 0x85, 0x01, 0x80, 0x84, 0xa9, 0x8c, 0x21, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45,
|
||||
0x4e, 0x44, 0xae, 0x42, 0x60, 0x82,
|
||||
}
|
||||
doc := core.ParsedDocument{
|
||||
Meta: core.DocMeta{Filename: "chart.png", MIME: "image/png"},
|
||||
Blocks: []core.DocBlock{{
|
||||
Type: core.BlockImageRef,
|
||||
Ref: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==",
|
||||
}},
|
||||
}
|
||||
msg := BuildCurrentUser(core.EffectiveSettings{}, fakeUser("看图"), []core.ParsedDocument{doc}, nil, &Report{})
|
||||
if len(msg.Images) != 1 || msg.Images[0].MIME != "image/png" || msg.Images[0].Data == "" {
|
||||
t.Fatalf("images %+v", msg.Images)
|
||||
}
|
||||
if !strings.Contains(msg.Content, "图像已随本消息发送") || strings.Contains(msg.Content, string(png)) {
|
||||
t.Fatalf("content %q", msg.Content)
|
||||
}
|
||||
|
||||
prev := core.Message{ID: "old", Role: core.RoleUser, Parts: []core.Part{
|
||||
{Type: core.PartText, Text: "上一张"},
|
||||
{Type: core.PartArtifactRef, Artifact: &core.ArtifactRef{ArtifactID: "a1", Name: "chart.png", Kind: "image"}},
|
||||
}}
|
||||
msgs, _ := AssembleInput(core.AgentProfile{}, core.EffectiveSettings{}, []core.Message{prev}, fakeUser("继续"), nil, nil, nil, nil, nil, func(core.ID) (core.Artifact, []byte, bool) {
|
||||
return core.Artifact{Name: "chart.png", MIME: "image/png"}, png, true
|
||||
}, AssembleOptions{})
|
||||
var hist string
|
||||
for _, m := range msgs {
|
||||
if strings.Contains(m.Content, "上一张") {
|
||||
hist = m.Content
|
||||
if len(m.Images) != 0 {
|
||||
t.Fatal("history must not resend image bytes")
|
||||
}
|
||||
}
|
||||
}
|
||||
if !strings.Contains(hist, "此前上传的图片:chart.png") {
|
||||
t.Fatalf("history placeholder missing: %q", hist)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParsePNGBecomesImageBlock(t *testing.T) {
|
||||
doc := Parse("chart.png", "image/png", onePixelPNG)
|
||||
if doc.Meta.Parser != "builtin:image" || len(doc.Blocks) != 1 || doc.Blocks[0].Ref == "" {
|
||||
t.Fatalf("%+v warnings=%v", doc.Meta, doc.Warnings)
|
||||
}
|
||||
if len(doc.Warnings) == 0 || !strings.Contains(doc.Warnings[0], "按图像发送") {
|
||||
t.Fatalf("warnings %v", doc.Warnings)
|
||||
}
|
||||
svg := Parse("logo.svg", "image/svg+xml", []byte("<svg/>"))
|
||||
if len(svg.Blocks) != 0 || len(svg.Warnings) == 0 || strings.Contains(svg.Warnings[0], "不走文本解析") {
|
||||
t.Fatalf("svg blocks=%d warnings=%v", len(svg.Blocks), svg.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
var onePixelPNG = []byte{
|
||||
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52,
|
||||
0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x02, 0x00, 0x00, 0x00, 0x90, 0x77, 0x53,
|
||||
0xde, 0x00, 0x00, 0x00, 0x0c, 0x49, 0x44, 0x41, 0x54, 0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0x50,
|
||||
0x0f, 0x00, 0x04, 0x85, 0x01, 0x80, 0x84, 0xa9, 0x8c, 0x21, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45,
|
||||
0x4e, 0x44, 0xae, 0x42, 0x60, 0x82,
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"agent-system-backend/internal/vision"
|
||||
core "agent-system-core"
|
||||
pdflib "github.com/ledongthuc/pdf"
|
||||
)
|
||||
@@ -87,7 +88,15 @@ func Parse(filename, mime string, content []byte) core.ParsedDocument {
|
||||
doc.Warnings = append(doc.Warnings, "旧版二进制格式 ."+ext+" 不解析:请转存为 ."+legacyOfficeTarget(ext)+" 后重传")
|
||||
default:
|
||||
if isImage(ext) {
|
||||
doc.Warnings = append(doc.Warnings, "图片不走文本解析(OCR 为 MinerU 深度档,远期)")
|
||||
block, warn, ok := vision.PrepareImage(filename, mime, content)
|
||||
if warn != "" {
|
||||
doc.Warnings = append(doc.Warnings, warn)
|
||||
}
|
||||
if ok {
|
||||
doc.Meta.Parser = "builtin:image"
|
||||
doc.Blocks = []core.DocBlock{block}
|
||||
doc.Meta.Confidence = 1
|
||||
}
|
||||
} else {
|
||||
// 未知类型按纯文本尝试(二进制内容会产生乱码 warning)。
|
||||
text := string(content)
|
||||
@@ -440,6 +449,7 @@ func ParsersDescription() []core.ParserDesc {
|
||||
{ID: "builtin:xlsx", Formats: []string{"xlsx", "xlsm"}, Source: core.SourceBuiltin},
|
||||
{ID: "builtin:pptx", Formats: []string{"pptx", "pptm", "ppsx"}, Source: core.SourceBuiltin},
|
||||
{ID: "builtin:legacy-office", Formats: []string{"doc", "xls", "ppt"}, Source: core.SourceBuiltin, Note: "不解析;提示转存现代格式后重传"},
|
||||
{ID: "builtin:image", Formats: []string{"png", "jpg", "jpeg", "gif", "webp"}, Source: core.SourceBuiltin, Note: "随当前轮消息以图像发送;历史轮只留文字占位"},
|
||||
{ID: "mineru", Formats: []string{"pdf", "png", "jpg"}, Source: core.SourcePlugin, Note: "远期可选:深度解析与交叉验证(默认不部署)"},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,9 @@ import (
|
||||
const OwnerDefault = store.OwnerDefault
|
||||
|
||||
const DefaultProfileID = "default"
|
||||
const FastProfileID = "fast"
|
||||
const ReasoningProfileID = "reasoning"
|
||||
const WritingProfileID = "writing"
|
||||
|
||||
// defaultSystemPrompt 默认画像不再携带常驻提示词:基座(身份/行为/谨慎/工具/风格)
|
||||
// 与环境段由提示词库组装(prompts 库 base.*/env.*);文档任务输出要求已技能化
|
||||
@@ -22,7 +25,7 @@ func DefaultProfile() core.AgentProfile {
|
||||
return core.AgentProfile{
|
||||
ID: DefaultProfileID,
|
||||
Name: "通用助手",
|
||||
Description: "内置默认画像:全部工具可用,HTTP 工具执行前确认(T0 基线)。",
|
||||
Description: "内置默认画像:全部工具可用,HTTP 工具执行前确认(T0 基线)。对话模型由系统设置决定。",
|
||||
Builtin: true,
|
||||
Source: "builtin",
|
||||
SystemPrompt: defaultSystemPrompt,
|
||||
@@ -35,19 +38,57 @@ func DefaultProfile() core.AgentProfile {
|
||||
}
|
||||
}
|
||||
|
||||
// BuiltinProfiles 是启动时幂等写入的全部内置画像。模型名是网关模型 ID,
|
||||
// 端点与密钥沿用系统设置。轻量档留空则回落设置里的轻量模型。
|
||||
func BuiltinProfiles() []core.AgentProfile {
|
||||
fast := DefaultProfile()
|
||||
fast.ID = FastProfileID
|
||||
fast.Name = "快速助手"
|
||||
fast.Description = "内置画像:对话和轻量任务都用 qwen3.8-flash,适合短问答和改写。"
|
||||
fast.Model = core.ModelRoute{Chat: "qwen3.8-flash", Lightweight: "qwen3.8-flash"}
|
||||
|
||||
reasoning := DefaultProfile()
|
||||
reasoning.ID = ReasoningProfileID
|
||||
reasoning.Name = "深度推理"
|
||||
reasoning.Description = "内置画像:对话用 ZHIPU/GLM-5.3-FlashX。分诊和校验仍走系统轻量模型。"
|
||||
reasoning.Model = core.ModelRoute{Chat: "ZHIPU/GLM-5.3-FlashX"}
|
||||
|
||||
writing := DefaultProfile()
|
||||
writing.ID = WritingProfileID
|
||||
writing.Name = "写作与文档"
|
||||
writing.Description = "内置画像:对话用 ZHIPU/GLM-5.3-Flash。分诊和校验仍走系统轻量模型。"
|
||||
writing.Model = core.ModelRoute{Chat: "ZHIPU/GLM-5.3-Flash"}
|
||||
|
||||
return []core.AgentProfile{DefaultProfile(), fast, reasoning, writing}
|
||||
}
|
||||
|
||||
func builtinID(id string) bool {
|
||||
for _, p := range BuiltinProfiles() {
|
||||
if p.ID == id {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type Service struct{ st *store.Store }
|
||||
|
||||
func New(st *store.Store) *Service { return &Service{st: st} }
|
||||
|
||||
// EnsureDefault 幂等写入内置画像;内置画像只读(Save 拒绝编辑),库中与内置定义
|
||||
// EnsureDefault 幂等写入全部内置画像;内置画像只读(Save 拒绝编辑),库中与内置定义
|
||||
// 有差异的行只可能来自更早版本的内置定义,因此差异时升级为当前内置定义。
|
||||
func (s *Service) EnsureDefault() error {
|
||||
if cur, err := s.st.GetProfile(OwnerDefault, DefaultProfileID); err == nil {
|
||||
if reflect.DeepEqual(cur, DefaultProfile()) {
|
||||
return nil
|
||||
for _, p := range BuiltinProfiles() {
|
||||
if cur, err := s.st.GetProfile(OwnerDefault, p.ID); err == nil {
|
||||
if reflect.DeepEqual(cur, p) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err := s.st.SaveProfile(p); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return s.st.SaveProfile(DefaultProfile())
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) List(owner string) ([]core.AgentProfile, error) { return s.st.ListProfiles(owner) }
|
||||
@@ -62,7 +103,7 @@ func (s *Service) Get(owner, id string) (core.AgentProfile, error) {
|
||||
}
|
||||
|
||||
func (s *Service) Save(owner string, p core.AgentProfile) error {
|
||||
if p.ID == DefaultProfileID {
|
||||
if builtinID(p.ID) {
|
||||
return errReadOnly("profiles: builtin profile is read-only; duplicate and modify")
|
||||
}
|
||||
if p.Tools.Allow == nil {
|
||||
@@ -72,7 +113,12 @@ func (s *Service) Save(owner string, p core.AgentProfile) error {
|
||||
return s.st.SaveProfile(p)
|
||||
}
|
||||
|
||||
func (s *Service) Delete(owner, id string) error { return s.st.DeleteProfile(owner, id) }
|
||||
func (s *Service) Delete(owner, id string) error {
|
||||
if builtinID(id) {
|
||||
return errReadOnly("profiles: builtin profile is read-only")
|
||||
}
|
||||
return s.st.DeleteProfile(owner, id)
|
||||
}
|
||||
|
||||
// SaveAccountChecked is the user-facing write path. Builtin profiles remain
|
||||
// read-only and the Store binds the write to the actor's effective account.
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package profiles
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"agent-system-backend/internal/store"
|
||||
core "agent-system-core"
|
||||
)
|
||||
|
||||
func TestBuiltinProfilesRoundTripAndReadOnly(t *testing.T) {
|
||||
st, err := store.OpenSQLite(filepath.Join(t.TempDir(), "p.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { st.Close() })
|
||||
svc := New(st)
|
||||
if err := svc.EnsureDefault(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := svc.EnsureDefault(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]string{
|
||||
DefaultProfileID: "",
|
||||
FastProfileID: "qwen3.8-flash",
|
||||
ReasoningProfileID: "ZHIPU/GLM-5.3-FlashX",
|
||||
WritingProfileID: "ZHIPU/GLM-5.3-Flash",
|
||||
}
|
||||
for id, model := range want {
|
||||
got, err := svc.Get(OwnerDefault, id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !got.Builtin || got.Model.Chat != model {
|
||||
t.Fatalf("%s: builtin=%v model=%q", id, got.Builtin, got.Model.Chat)
|
||||
}
|
||||
if err := svc.Save("local", core.AgentProfile{ID: id, Name: "改"}); err == nil {
|
||||
t.Fatalf("save %s should be rejected", id)
|
||||
}
|
||||
if err := svc.Delete("local", id); err == nil {
|
||||
t.Fatalf("delete %s should be rejected", id)
|
||||
}
|
||||
}
|
||||
listed, err := svc.List(OwnerDefault)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(listed) != len(want) {
|
||||
t.Fatalf("listed %d", len(listed))
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"agent-system-backend/internal/ctxasm"
|
||||
"agent-system-backend/internal/prompts"
|
||||
"agent-system-backend/internal/quality"
|
||||
"agent-system-backend/internal/vision"
|
||||
core "agent-system-core"
|
||||
)
|
||||
|
||||
@@ -228,7 +229,8 @@ func weekdayCN(t time.Time) string {
|
||||
func (l *LLMRunner) Execute(rc RunCtx) error {
|
||||
eff := rc.Settings()
|
||||
prof := rc.Profile()
|
||||
route := ResolveChatModel(prof, eff)
|
||||
hasImages := vision.HasVisualInput(rc.Attachments(), currentUserText(rc))
|
||||
route := ResolveVisionModel(prof, eff, hasImages)
|
||||
if route.Model == "" || route.Base == "" {
|
||||
return fmt.Errorf("llm: 主模型未配置(当前解析:model=%q base=%q;检查 settings.models.chat 与画像 ModelRoute;演示模式可开 runtime.mock)", route.Model, route.Base)
|
||||
}
|
||||
@@ -275,6 +277,7 @@ func (l *LLMRunner) Execute(rc RunCtx) error {
|
||||
Surfaces: rc.Surfaces(),
|
||||
ThinkReplay: thinkReplayEnabled(eff, route.Model),
|
||||
})
|
||||
vision.AttachLinkedImages(rc.Context(), msgs)
|
||||
tail := ctxasm.Tail{
|
||||
Instructions: eff.GetString("context.userInstructions", ""),
|
||||
History: hist,
|
||||
|
||||
@@ -126,6 +126,15 @@ func (t directTool) Execute(ctx context.Context, args json.RawMessage) (agenttoo
|
||||
return t.run(ctx, args), nil
|
||||
}
|
||||
|
||||
func messagesHaveImages(msgs []agentllm.Message) bool {
|
||||
for _, m := range msgs {
|
||||
if len(m.Images) > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (l *LLMRunner) runDirect(rc RunCtx, stream agentllm.StreamFunc, model string, msgs []agentllm.Message, decls []agentllm.ToolDecl, streamOpts agentllm.StreamOptions, cardflow bool, eff core.EffectiveSettings, maxTurns, maxTools int, refresh turnRefresh) error {
|
||||
st := &directRun{l: l, rc: rc, eff: eff, cardflow: cardflow, started: l.now(), inflight: map[string]*toolSlot{}}
|
||||
// 记忆端口(N3 WP5):runCtx 暴露 store 时接线(注入由循环按请求完成)。
|
||||
@@ -161,7 +170,7 @@ func (l *LLMRunner) runDirect(rc RunCtx, stream agentllm.StreamFunc, model strin
|
||||
break
|
||||
}
|
||||
}
|
||||
stream = agentllm.WithFallback(stream, ChatFallbacks(eff), st.noteModelSwitch)
|
||||
stream = agentllm.WithFallback(stream, ChatFallbacksFor(st.eff, messagesHaveImages(msgs)), st.noteModelSwitch)
|
||||
defer func() {
|
||||
if sink, ok := rc.(interface{ AddUsage(core.Usage) }); ok {
|
||||
st.switchMu.Lock()
|
||||
|
||||
@@ -3,6 +3,7 @@ package runtime
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"agent-system-backend/internal/vision"
|
||||
core "agent-system-core"
|
||||
)
|
||||
|
||||
@@ -61,12 +62,48 @@ func ResolveChatModel(prof core.AgentProfile, eff core.EffectiveSettings) Resolv
|
||||
Key: overlayKey(prof.Owner, "models.chat", eff.GetString("models.chat.apiKey", ""))}
|
||||
}
|
||||
|
||||
// ChatFallbacks 解析 models.chat.fallbacks。默认空,不改变现有模型选择。
|
||||
func ChatFallbacks(eff core.EffectiveSettings) []string {
|
||||
raw := strings.TrimSpace(eff.GetString("models.chat.fallbacks", ""))
|
||||
if raw == "" {
|
||||
return nil
|
||||
// ResolveVisionModel 在当前轮含图片时核对视觉清单。清单内的模型保持原选择;
|
||||
// 不在清单内则这一轮改用 models.vision.model,端点和密钥仍沿用主模型。
|
||||
// 没有图片时与 ResolveChatModel 相同。
|
||||
func ResolveVisionModel(prof core.AgentProfile, eff core.EffectiveSettings, hasImages bool) ResolvedModel {
|
||||
chat := ResolveChatModel(prof, eff)
|
||||
if !hasImages || modelCapable(chat.Model, VisionCapable(eff)) {
|
||||
return chat
|
||||
}
|
||||
name := strings.TrimSpace(eff.GetString("models.vision.model", ""))
|
||||
if name == "" {
|
||||
name = vision.DefaultVisionModel
|
||||
}
|
||||
if name == chat.Model {
|
||||
return chat
|
||||
}
|
||||
chat.Model = name
|
||||
return chat
|
||||
}
|
||||
|
||||
// VisionCapable 解析 models.vision.capable。空字符串用内置默认清单。
|
||||
func VisionCapable(eff core.EffectiveSettings) []string {
|
||||
raw := strings.TrimSpace(eff.GetString("models.vision.capable", ""))
|
||||
if raw == "" {
|
||||
raw = vision.DefaultVisionCapable
|
||||
}
|
||||
return splitModelList(raw)
|
||||
}
|
||||
|
||||
func modelCapable(name string, capable []string) bool {
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
return false
|
||||
}
|
||||
for _, item := range capable {
|
||||
if item == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func splitModelList(raw string) []string {
|
||||
var out []string
|
||||
seen := map[string]bool{}
|
||||
for _, part := range strings.Split(raw, ",") {
|
||||
@@ -80,6 +117,27 @@ func ChatFallbacks(eff core.EffectiveSettings) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// ChatFallbacks 解析 models.chat.fallbacks。默认空,不改变现有模型选择。
|
||||
func ChatFallbacks(eff core.EffectiveSettings) []string {
|
||||
return splitModelList(eff.GetString("models.chat.fallbacks", ""))
|
||||
}
|
||||
|
||||
// ChatFallbacksFor 在图片轮只保留视觉清单内的回退模型,避免退到不能看图的模型。
|
||||
func ChatFallbacksFor(eff core.EffectiveSettings, hasImages bool) []string {
|
||||
all := ChatFallbacks(eff)
|
||||
if !hasImages {
|
||||
return all
|
||||
}
|
||||
capable := VisionCapable(eff)
|
||||
var out []string
|
||||
for _, name := range all {
|
||||
if modelCapable(name, capable) {
|
||||
out = append(out, name)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ResolveLightweightModel 解析轻量模型:画像 route > 设置层轻量档 > 回落主档。
|
||||
func ResolveLightweightModel(prof core.AgentProfile, eff core.EffectiveSettings) ResolvedModel {
|
||||
m := eff.GetString("models.lightweight.model", "")
|
||||
|
||||
@@ -58,3 +58,35 @@ func TestChatFallbacksDefaultEmpty(t *testing.T) {
|
||||
t.Fatalf("%#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveVisionModel(t *testing.T) {
|
||||
eff := effOf(map[string]any{"models": map[string]any{"chat": map[string]any{
|
||||
"model": "deepseek-v4.1-flash", "baseUrl": "https://gw.example/v1", "apiKey": "k",
|
||||
"fallbacks": "plain-text,ZHIPU/GLM-5.3-FlashX",
|
||||
}, "vision": map[string]any{
|
||||
"model": "qwen3.8-flash",
|
||||
"capable": "deepseek-v4.1-flash,qwen3.8-flash,ZHIPU/GLM-5.3-FlashX",
|
||||
}}})
|
||||
prof := core.AgentProfile{}
|
||||
got := ResolveVisionModel(prof, eff, false)
|
||||
if got.Model != "deepseek-v4.1-flash" {
|
||||
t.Fatalf("no image should keep chat model: %+v", got)
|
||||
}
|
||||
got = ResolveVisionModel(prof, eff, true)
|
||||
if got.Model != "deepseek-v4.1-flash" || got.Base != "https://gw.example/v1" || got.Key != "k" {
|
||||
t.Fatalf("capable model should stay: %+v", got)
|
||||
}
|
||||
prof.Model.Chat = "plain-text"
|
||||
got = ResolveVisionModel(prof, eff, true)
|
||||
if got.Model != "qwen3.8-flash" || got.Base != "https://gw.example/v1" || got.Key != "k" {
|
||||
t.Fatalf("incapable model should swap, keeping endpoint: %+v", got)
|
||||
}
|
||||
fb := ChatFallbacksFor(eff, true)
|
||||
if len(fb) != 1 || fb[0] != "ZHIPU/GLM-5.3-FlashX" {
|
||||
t.Fatalf("image fallbacks %#v", fb)
|
||||
}
|
||||
fb = ChatFallbacksFor(eff, false)
|
||||
if len(fb) != 2 || fb[0] != "plain-text" {
|
||||
t.Fatalf("text fallbacks %#v", fb)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,8 +39,12 @@ func SystemDefaults() map[string]any {
|
||||
"fallbacks": "", // 逗号分隔;空 = 不回退
|
||||
},
|
||||
"lightweight": map[string]any{"baseUrl": "", "apiKey": "", "model": ""},
|
||||
"embedding": map[string]any{"baseUrl": "", "apiKey": "", "model": ""},
|
||||
"rerank": map[string]any{"baseUrl": "", "apiKey": "", "model": ""},
|
||||
"vision": map[string]any{
|
||||
"model": "qwen3.8-flash",
|
||||
"capable": "deepseek-v4.1-flash,qwen3.8-flash,ZHIPU/GLM-5.3-FlashX,ZHIPU/GLM-5.3-Flash",
|
||||
},
|
||||
"embedding": map[string]any{"baseUrl": "", "apiKey": "", "model": ""},
|
||||
"rerank": map[string]any{"baseUrl": "", "apiKey": "", "model": ""},
|
||||
},
|
||||
"tools": map[string]any{
|
||||
"tavily": map[string]any{"apiKeys": "", "endpoint": ""}, // 网络搜索(逗号分隔密钥池;空 = 不注册搜索工具)
|
||||
@@ -100,6 +104,8 @@ func Schema() []core.SettingDesc {
|
||||
{Key: "models.lightweight.baseUrl", Group: "models", Label: "轻量模型 Base URL", Type: core.SettingString, Default: "", Effect: core.EffectHot, Scope: core.SettingScopeSystem, Description: "标题生成/澄清等轻任务;空则回落主模型"},
|
||||
{Key: "models.lightweight.apiKey", Group: "models", Label: "轻量模型 API Key", Type: core.SettingSecret, Default: "", Effect: core.EffectHot, Scope: core.SettingScopeSystem},
|
||||
{Key: "models.lightweight.model", Group: "models", Label: "轻量模型名称", Type: core.SettingString, Default: "", Effect: core.EffectNewRun, Scope: core.SettingScopeSystem},
|
||||
{Key: "models.vision.model", Group: "models", Label: "视觉兜底模型", Type: core.SettingString, Default: "qwen3.8-flash", Effect: core.EffectNewRun, Scope: core.SettingScopeSystem, Description: "当前轮含图片且主模型不在视觉清单内时改用此模型。端点与密钥沿用主模型"},
|
||||
{Key: "models.vision.capable", Group: "models", Label: "可看图模型", Type: core.SettingString, Default: "deepseek-v4.1-flash,qwen3.8-flash,ZHIPU/GLM-5.3-FlashX,ZHIPU/GLM-5.3-Flash", Effect: core.EffectNewRun, Scope: core.SettingScopeSystem, Description: "逗号分隔。当前轮有图片时,主模型和回退链只保留清单内的名字。空则使用内置默认清单"},
|
||||
{Key: "models.lightweight.maxOutputTokens", Group: "models", Label: "轻量模型输出上限", Type: core.SettingNumber, Default: 0, Effect: core.EffectNewRun, Scope: core.SettingScopeSystem, Description: "单次回复的 max_tokens。0 = 不限制,由模型服务端决定"},
|
||||
{Key: "models.embedding.baseUrl", Group: "models", Label: "Embedding Base URL", Type: core.SettingString, Default: "", Effect: core.EffectHot, Scope: core.SettingScopeSystem, Description: "RAG/记忆检索用(向量管线远期启用)"},
|
||||
{Key: "models.embedding.apiKey", Group: "models", Label: "Embedding API Key", Type: core.SettingSecret, Default: "", Effect: core.EffectHot, Scope: core.SettingScopeSystem},
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
package vision
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
agentllm "agent-native/runtime/llm"
|
||||
)
|
||||
|
||||
var (
|
||||
errPrivateAddress = errors.New("地址不在允许范围")
|
||||
errNotImage = errors.New("响应不是受支持的图片")
|
||||
errTooLarge = errors.New("超过单图大小上限")
|
||||
)
|
||||
|
||||
// FetchImage 下载一张公网图片。拒绝回环、私网、链路本地和云元数据地址,
|
||||
// 并把连接钉在解析到的公网 IP 上,避免 DNS 重绑定。不走环境变量里的代理。
|
||||
func FetchImage(ctx context.Context, rawURL string) (agentllm.Image, error) {
|
||||
if err := validateImageURL(ctx, rawURL); err != nil {
|
||||
return agentllm.Image{}, err
|
||||
}
|
||||
client := &http.Client{
|
||||
Timeout: 20 * time.Second,
|
||||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= 3 {
|
||||
return errors.New("重定向过多")
|
||||
}
|
||||
if err := validateImageURL(req.Context(), req.URL.String()); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Transport: &http.Transport{
|
||||
Proxy: nil,
|
||||
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
host, port, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ip, err := pickPublicIP(ctx, host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var d net.Dialer
|
||||
d.Timeout = 10 * time.Second
|
||||
return d.DialContext(ctx, network, net.JoinHostPort(ip.String(), port))
|
||||
},
|
||||
ResponseHeaderTimeout: 10 * time.Second,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
},
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
|
||||
if err != nil {
|
||||
return agentllm.Image{}, err
|
||||
}
|
||||
req.Header.Set("User-Agent", "agent-system-vision/1.0")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
if errors.Is(err, errPrivateAddress) {
|
||||
return agentllm.Image{}, errPrivateAddress
|
||||
}
|
||||
return agentllm.Image{}, fmt.Errorf("下载失败")
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return agentllm.Image{}, fmt.Errorf("下载失败")
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, MaxImageBytes+1))
|
||||
if err != nil {
|
||||
return agentllm.Image{}, fmt.Errorf("下载失败")
|
||||
}
|
||||
if len(body) > MaxImageBytes {
|
||||
return agentllm.Image{}, errTooLarge
|
||||
}
|
||||
kind := MIMEOf(body)
|
||||
if kind == "" || !contentTypeOK(resp.Header.Get("Content-Type"), kind) {
|
||||
return agentllm.Image{}, errNotImage
|
||||
}
|
||||
// 只回传字节。URL 留给网关自己去拉会绕过这里的地址校验。
|
||||
return agentllm.Image{
|
||||
MIME: kind,
|
||||
Data: base64.StdEncoding.EncodeToString(body),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func publicReason(err error) string {
|
||||
switch {
|
||||
case err == nil:
|
||||
return ""
|
||||
case errors.Is(err, errPrivateAddress):
|
||||
return errPrivateAddress.Error()
|
||||
case errors.Is(err, errNotImage):
|
||||
return errNotImage.Error()
|
||||
case errors.Is(err, errTooLarge):
|
||||
return errTooLarge.Error()
|
||||
default:
|
||||
return "下载失败"
|
||||
}
|
||||
}
|
||||
|
||||
func validateImageURL(ctx context.Context, raw string) error {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil || u.Host == "" {
|
||||
return errPrivateAddress
|
||||
}
|
||||
if u.User != nil {
|
||||
return errPrivateAddress
|
||||
}
|
||||
switch strings.ToLower(u.Scheme) {
|
||||
case "http", "https":
|
||||
default:
|
||||
return errPrivateAddress
|
||||
}
|
||||
if _, err := pickPublicIP(ctx, u.Hostname()); err != nil {
|
||||
return errPrivateAddress
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func pickPublicIP(ctx context.Context, host string) (net.IP, error) {
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
if err := checkIP(ip); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ip, nil
|
||||
}
|
||||
if host == "" || strings.EqualFold(host, "localhost") {
|
||||
return nil, errPrivateAddress
|
||||
}
|
||||
addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
|
||||
if err != nil || len(addrs) == 0 {
|
||||
return nil, errPrivateAddress
|
||||
}
|
||||
var first net.IP
|
||||
for _, addr := range addrs {
|
||||
if err := checkIP(addr.IP); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if first == nil {
|
||||
first = addr.IP
|
||||
}
|
||||
}
|
||||
if first == nil {
|
||||
return nil, errPrivateAddress
|
||||
}
|
||||
return first, nil
|
||||
}
|
||||
|
||||
var (
|
||||
cgnat = mustCIDR("100.64.0.0/10")
|
||||
)
|
||||
|
||||
func mustCIDR(s string) *net.IPNet {
|
||||
_, n, err := net.ParseCIDR(s)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func checkIP(ip net.IP) error {
|
||||
if ip == nil {
|
||||
return errPrivateAddress
|
||||
}
|
||||
if v4 := ip.To4(); v4 != nil {
|
||||
ip = v4
|
||||
}
|
||||
switch {
|
||||
case ip.IsLoopback(), ip.IsPrivate(), ip.IsLinkLocalUnicast(), ip.IsLinkLocalMulticast(),
|
||||
ip.IsMulticast(), ip.IsUnspecified(), cgnat.Contains(ip):
|
||||
return errPrivateAddress
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func contentTypeOK(header, sniffed string) bool {
|
||||
ct := strings.ToLower(strings.TrimSpace(strings.Split(header, ";")[0]))
|
||||
switch ct {
|
||||
case "", "application/octet-stream", "binary/octet-stream":
|
||||
return sniffed != ""
|
||||
default:
|
||||
return ct == sniffed
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
package vision
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
agentllm "agent-native/runtime/llm"
|
||||
core "agent-system-core"
|
||||
)
|
||||
|
||||
// PrepareImage 把上传的图片收成一条 image-ref。Ref 是标准 base64(无换行)。
|
||||
// 不支持的类型和超限图片返回警告且 ok 为 false,调用方必须把警告展示出来。
|
||||
func PrepareImage(filename, mime string, content []byte) (core.DocBlock, string, bool) {
|
||||
ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(filename), "."))
|
||||
kind, known := sendableExt[ext]
|
||||
if !known {
|
||||
return core.DocBlock{}, fmt.Sprintf("图片 .%s 不作为图像发送(请转为 png、jpeg、gif 或 webp)", ext), false
|
||||
}
|
||||
if len(content) == 0 {
|
||||
return core.DocBlock{}, "图片内容为空,本轮不发送图像", false
|
||||
}
|
||||
if len(content) > MaxImageBytes {
|
||||
return core.DocBlock{}, fmt.Sprintf("图片超过 %dMB,本轮不发送图像(请压缩后重传)", MaxImageBytes>>20), false
|
||||
}
|
||||
detected := sniffImage(content)
|
||||
if detected == "" || detected != kind {
|
||||
return core.DocBlock{}, "图片内容与扩展名不符,本轮不发送图像", false
|
||||
}
|
||||
if mime != "" && !strings.EqualFold(strings.TrimSpace(strings.Split(mime, ";")[0]), kind) && !strings.EqualFold(mime, "application/octet-stream") {
|
||||
return core.DocBlock{}, "图片类型与内容不符,本轮不发送图像", false
|
||||
}
|
||||
return core.DocBlock{
|
||||
Type: core.BlockImageRef,
|
||||
Ref: base64.StdEncoding.EncodeToString(content),
|
||||
Text: filename,
|
||||
}, "图片按图像发送给模型,不转成文字", true
|
||||
}
|
||||
|
||||
// ImagesFromDoc 取出文档里已编码的图片。MIME 来自文件名,不信任外部声明。
|
||||
func ImagesFromDoc(doc core.ParsedDocument) []agentllm.Image {
|
||||
ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(doc.Meta.Filename), "."))
|
||||
kind := sendableExt[ext]
|
||||
if kind == "" {
|
||||
return nil
|
||||
}
|
||||
var out []agentllm.Image
|
||||
for _, b := range doc.Blocks {
|
||||
if b.Type != core.BlockImageRef || strings.TrimSpace(b.Ref) == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, agentllm.Image{MIME: kind, Data: b.Ref})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// HasVisualInput 判断这一轮是否带图:已解析的图片附件,或正文里的图片链接。
|
||||
func HasVisualInput(docs []core.ParsedDocument, text string) bool {
|
||||
for _, doc := range docs {
|
||||
if len(ImagesFromDoc(doc)) > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return len(ImageLinks(text)) > 0
|
||||
}
|
||||
|
||||
// Admit 按张数和总字节数收下一批图片。被拒绝的每一项都有一句说明。
|
||||
func Admit(existing []agentllm.Image, incoming []agentllm.Image) (kept []agentllm.Image, notes []string) {
|
||||
count, total := usage(existing)
|
||||
for _, img := range incoming {
|
||||
n := decodedLen(img.Data)
|
||||
switch {
|
||||
case n <= 0:
|
||||
notes = append(notes, "一张图片没有可用内容,已跳过")
|
||||
case n > MaxImageBytes:
|
||||
notes = append(notes, fmt.Sprintf("一张图片超过 %dMB,已跳过", MaxImageBytes>>20))
|
||||
case count >= MaxImages:
|
||||
notes = append(notes, fmt.Sprintf("本轮图片超过 %d 张,其余未发送", MaxImages))
|
||||
return kept, notes
|
||||
case total+n > MaxTotalBytes:
|
||||
notes = append(notes, fmt.Sprintf("本轮图片合计超过 %dMB,其余未发送", MaxTotalBytes>>20))
|
||||
return kept, notes
|
||||
default:
|
||||
kept = append(kept, img)
|
||||
count++
|
||||
total += n
|
||||
}
|
||||
}
|
||||
return kept, notes
|
||||
}
|
||||
|
||||
func usage(images []agentllm.Image) (count, total int) {
|
||||
for _, img := range images {
|
||||
n := decodedLen(img.Data)
|
||||
if n > 0 {
|
||||
count++
|
||||
total += n
|
||||
}
|
||||
}
|
||||
return count, total
|
||||
}
|
||||
|
||||
func decodedLen(data string) int {
|
||||
raw, err := base64.StdEncoding.DecodeString(data)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return len(raw)
|
||||
}
|
||||
|
||||
var sendableExt = map[string]string{
|
||||
"png": "image/png",
|
||||
"jpg": "image/jpeg",
|
||||
"jpeg": "image/jpeg",
|
||||
"gif": "image/gif",
|
||||
"webp": "image/webp",
|
||||
}
|
||||
|
||||
func sniffImage(b []byte) string {
|
||||
switch {
|
||||
case len(b) >= 8 && string(b[:8]) == "\x89PNG\r\n\x1a\n":
|
||||
return "image/png"
|
||||
case len(b) >= 3 && b[0] == 0xFF && b[1] == 0xD8 && b[2] == 0xFF:
|
||||
return "image/jpeg"
|
||||
case len(b) >= 6 && (string(b[:6]) == "GIF87a" || string(b[:6]) == "GIF89a"):
|
||||
return "image/gif"
|
||||
case len(b) >= 12 && string(b[:4]) == "RIFF" && string(b[8:12]) == "WEBP":
|
||||
return "image/webp"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// MIMEOf 返回嗅探结果,供下载后的响应校验。
|
||||
func MIMEOf(content []byte) string { return sniffImage(content) }
|
||||
@@ -0,0 +1,90 @@
|
||||
package vision
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
agentllm "agent-native/runtime/llm"
|
||||
)
|
||||
|
||||
// 只认路径以图片扩展名结尾的 http(s) 链接。不根据任意 URL 去探测,
|
||||
// 避免把用户消息里的普通网页都拿去下载。
|
||||
var imageURLRe = regexp.MustCompile(`(?i)https?://[^\s<>"',。、;:]+\.(?:png|jpe?g|gif|webp)(?:\?[^\s<>"',。、;:]*)?`)
|
||||
|
||||
// ImageLinks 按出现顺序返回去重后的图片链接。
|
||||
func ImageLinks(text string) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, raw := range imageURLRe.FindAllString(text, -1) {
|
||||
u := strings.TrimRight(raw, ".,;:)]>\"'")
|
||||
if u == "" || seen[u] {
|
||||
continue
|
||||
}
|
||||
seen[u] = true
|
||||
out = append(out, u)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// AttachLinkedImages 下载最后一条用户消息里的图片链接,并写进该消息的 Images。
|
||||
// 下载失败、超限或地址被拒绝时,把原因追加到正文,不静默丢掉。
|
||||
func AttachLinkedImages(ctx context.Context, msgs []agentllm.Message) []string {
|
||||
idx := -1
|
||||
for i := len(msgs) - 1; i >= 0; i-- {
|
||||
if msgs[i].Role == agentllm.RoleUser {
|
||||
idx = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if idx < 0 {
|
||||
return nil
|
||||
}
|
||||
links := ImageLinks(msgs[idx].Content)
|
||||
if len(links) == 0 {
|
||||
return nil
|
||||
}
|
||||
var notes []string
|
||||
var incoming []agentllm.Image
|
||||
byData := map[string]string{}
|
||||
for _, link := range links {
|
||||
img, err := FetchImage(ctx, link)
|
||||
if err != nil {
|
||||
notes = append(notes, fmt.Sprintf("%s 未能下载:%s", link, publicReason(err)))
|
||||
continue
|
||||
}
|
||||
incoming = append(incoming, img)
|
||||
byData[img.Data] = link
|
||||
}
|
||||
kept, dropped := Admit(msgs[idx].Images, incoming)
|
||||
msgs[idx].Images = append(msgs[idx].Images, kept...)
|
||||
sent := map[string]bool{}
|
||||
for _, img := range kept {
|
||||
if link := byData[img.Data]; link != "" {
|
||||
notes = append(notes, link+" 已随消息发送")
|
||||
sent[img.Data] = true
|
||||
}
|
||||
}
|
||||
for _, img := range incoming {
|
||||
if !sent[img.Data] {
|
||||
if link := byData[img.Data]; link != "" {
|
||||
notes = append(notes, link+" 未发送")
|
||||
}
|
||||
}
|
||||
}
|
||||
notes = append(notes, dropped...)
|
||||
if len(notes) == 0 {
|
||||
return nil
|
||||
}
|
||||
msgs[idx].Content += "\n\n[图片链接]\n" + strings.Join(prefixLines(notes), "\n") + "\n"
|
||||
return notes
|
||||
}
|
||||
|
||||
func prefixLines(notes []string) []string {
|
||||
out := make([]string, len(notes))
|
||||
for i, n := range notes {
|
||||
out[i] = "- " + n
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
// Package vision 负责「这一轮要不要把图发给模型」:能力清单、图片字节预算、
|
||||
// 以及用户粘贴的图片链接的受控下载。模型名和端点仍由 runtime 的路由解析。
|
||||
package vision
|
||||
|
||||
// 默认可看图清单与兜底模型。网关路由变化时改设置 models.vision.capable,
|
||||
// 不必改代码。空设置回落到这里。
|
||||
const (
|
||||
DefaultVisionModel = "qwen3.8-flash"
|
||||
DefaultVisionCapable = "deepseek-v4.1-flash,qwen3.8-flash,ZHIPU/GLM-5.3-FlashX,ZHIPU/GLM-5.3-Flash"
|
||||
)
|
||||
|
||||
// 单图与单轮上限。按网关实测的小图请求体留出余量:base64 约膨胀 4/3,
|
||||
// 8MB 原图约 11MB 请求体,单轮合计 20MB 原图。
|
||||
const (
|
||||
MaxImageBytes = 8 << 20
|
||||
MaxImages = 8
|
||||
MaxTotalBytes = 20 << 20
|
||||
)
|
||||
@@ -0,0 +1,120 @@
|
||||
package vision
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
agentllm "agent-native/runtime/llm"
|
||||
core "agent-system-core"
|
||||
)
|
||||
|
||||
func TestCheckIP(t *testing.T) {
|
||||
blocked := []string{"127.0.0.1", "10.1.2.3", "192.168.1.1", "172.16.0.1", "169.254.169.254", "100.64.0.1", "::1", "fc00::1", "0.0.0.0"}
|
||||
for _, s := range blocked {
|
||||
if err := checkIP(net.ParseIP(s)); err == nil {
|
||||
t.Errorf("%s should be blocked", s)
|
||||
}
|
||||
}
|
||||
if err := checkIP(net.ParseIP("8.8.8.8")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := checkIP(net.ParseIP("1.1.1.1")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateImageURLRejectsUnsafeTargets(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for _, raw := range []string{
|
||||
"http://127.0.0.1/a.png",
|
||||
"http://169.254.169.254/latest/a.png",
|
||||
"http://10.0.0.8/a.jpg",
|
||||
"http://user:pass@8.8.8.8/a.png",
|
||||
"file:///tmp/a.png",
|
||||
"http://localhost/a.png",
|
||||
} {
|
||||
if err := validateImageURL(ctx, raw); err == nil {
|
||||
t.Errorf("accepted %s", raw)
|
||||
}
|
||||
}
|
||||
if err := validateImageURL(ctx, "https://8.8.8.8/a.png"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchImageRejectsLoopbackServer(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
_, _ = w.Write(onePixelPNG)
|
||||
}))
|
||||
defer srv.Close()
|
||||
_, err := FetchImage(context.Background(), srv.URL+"/pixel.png")
|
||||
if err == nil || !strings.Contains(err.Error(), "允许范围") {
|
||||
t.Fatalf("err = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageLinksAndAttachNote(t *testing.T) {
|
||||
text := "看这张 http://127.0.0.1/a.png。还有 https://cdn.example.com/b.JPG?x=1 和重复 http://127.0.0.1/a.png"
|
||||
links := ImageLinks(text)
|
||||
if len(links) != 2 || links[0] != "http://127.0.0.1/a.png" || !strings.Contains(links[1], "b.JPG") {
|
||||
t.Fatalf("%#v", links)
|
||||
}
|
||||
msgs := []agentllm.Message{{Role: agentllm.RoleUser, Content: "请看 http://127.0.0.1/secret.png"}}
|
||||
notes := AttachLinkedImages(context.Background(), msgs)
|
||||
if len(notes) == 0 || len(msgs[0].Images) != 0 {
|
||||
t.Fatalf("notes=%v images=%d", notes, len(msgs[0].Images))
|
||||
}
|
||||
if !strings.Contains(msgs[0].Content, "地址不在允许范围") {
|
||||
t.Fatalf("content missing reason: %s", msgs[0].Content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareImageAndBudget(t *testing.T) {
|
||||
block, warn, ok := PrepareImage("pixel.png", "image/png", onePixelPNG)
|
||||
if !ok || !strings.Contains(warn, "按图像发送") || block.Type != core.BlockImageRef || block.Ref == "" {
|
||||
t.Fatalf("ok=%v warn=%q block=%+v", ok, warn, block)
|
||||
}
|
||||
if _, warn, ok := PrepareImage("icon.svg", "image/svg+xml", []byte("<svg/>")); ok || !strings.Contains(warn, "不作为图像发送") {
|
||||
t.Fatalf("svg ok=%v warn=%q", ok, warn)
|
||||
}
|
||||
big := make([]byte, MaxImageBytes+1)
|
||||
if _, warn, ok := PrepareImage("big.png", "image/png", big); ok || !strings.Contains(warn, "超过") {
|
||||
t.Fatalf("big ok=%v warn=%q", ok, warn)
|
||||
}
|
||||
if _, warn, ok := PrepareImage("bad.png", "image/png", []byte("not a png")); ok || !strings.Contains(warn, "不符") {
|
||||
t.Fatalf("bad ok=%v warn=%q", ok, warn)
|
||||
}
|
||||
|
||||
img := agentllm.Image{MIME: "image/png", Data: block.Ref}
|
||||
kept, notes := Admit(nil, []agentllm.Image{img})
|
||||
if len(kept) != 1 || len(notes) != 0 {
|
||||
t.Fatalf("kept=%d notes=%v", len(kept), notes)
|
||||
}
|
||||
many := make([]agentllm.Image, MaxImages+1)
|
||||
for i := range many {
|
||||
many[i] = img
|
||||
}
|
||||
kept, notes = Admit(nil, many)
|
||||
if len(kept) != MaxImages || len(notes) == 0 {
|
||||
t.Fatalf("kept=%d notes=%v", len(kept), notes)
|
||||
}
|
||||
raw, err := base64.StdEncoding.DecodeString(block.Ref)
|
||||
if err != nil || len(raw) != len(onePixelPNG) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1x1 PNG。
|
||||
var onePixelPNG = []byte{
|
||||
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52,
|
||||
0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x02, 0x00, 0x00, 0x00, 0x90, 0x77, 0x53,
|
||||
0xde, 0x00, 0x00, 0x00, 0x0c, 0x49, 0x44, 0x41, 0x54, 0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0x50,
|
||||
0x0f, 0x00, 0x04, 0x85, 0x01, 0x80, 0x84, 0xa9, 0x8c, 0x21, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45,
|
||||
0x4e, 0x44, 0xae, 0x42, 0x60, 0x82,
|
||||
}
|
||||
Reference in New Issue
Block a user