diff --git a/internal/ctxasm/ctxasm.go b/internal/ctxasm/ctxasm.go
index 010a3d5..cba242d 100644
--- a/internal/ctxasm/ctxasm.go
+++ b/internal/ctxasm/ctxasm.go
@@ -4,12 +4,14 @@ package ctxasm
import (
"encoding/json"
"fmt"
+ "path/filepath"
"regexp"
"strings"
"time"
"unicode/utf8"
agentllm "agent-native/runtime/llm"
+ "agent-system-backend/internal/vision"
core "agent-system-core"
)
@@ -134,6 +136,7 @@ func AssembleInput(prof core.AgentProfile, eff core.EffectiveSettings, history [
// 此处与整段组装共用同一实现,保证两条路径的当前轮语义一致。
func BuildCurrentUser(eff core.EffectiveSettings, currentUser core.Message, attachments []core.ParsedDocument, knowledge []core.RetrievalHit, rep *Report) agentllm.Message {
var sb strings.Builder
+ var images []agentllm.Image
for _, p := range currentUser.Parts {
if p.Type == core.PartText {
sb.WriteString(p.Text)
@@ -146,6 +149,22 @@ func BuildCurrentUser(eff core.EffectiveSettings, currentUser core.Message, atta
}
sb.WriteString("\n\n[附件](用户上传文档的解析全文,按原文顺序保留标题/列表/表格;超长时文末标注截断。请依据文档内容完成任务,输出完整不省略)\n")
for _, doc := range attachments {
+ if pics := vision.ImagesFromDoc(doc); len(pics) > 0 {
+ kept, notes := vision.Admit(images, pics)
+ images = append(images, kept...)
+ if len(kept) > 0 {
+ fmt.Fprintf(&sb, "### %s\n[图片 %s,图像已随本消息发送]\n", doc.Meta.Filename, doc.Meta.Filename)
+ } else {
+ fmt.Fprintf(&sb, "### %s\n[图片 %s,本轮未发送图像]\n", doc.Meta.Filename, doc.Meta.Filename)
+ }
+ for _, note := range notes {
+ fmt.Fprintf(&sb, "%s\n", note)
+ if rep != nil {
+ rep.Truncations = append(rep.Truncations, fmt.Sprintf("附件 %s:%s", doc.Meta.Filename, note))
+ }
+ }
+ continue
+ }
text := doc.PlainText()
if runes := []rune(text); len(runes) > limit {
text = string(runes[:limit]) + "…(附件超长截断)"
@@ -168,7 +187,7 @@ func BuildCurrentUser(eff core.EffectiveSettings, currentUser core.Message, atta
rep.Items = append(rep.Items, fmt.Sprintf("knowledge(%d)", len(knowledge)))
}
}
- return agentllm.Message{Role: agentllm.RoleUser, Content: sb.String()}
+ return agentllm.Message{Role: agentllm.RoleUser, Content: sb.String(), Images: images}
}
// maxToolArgRunes 是回放进模型的单次工具参数上限。超限时截断字符串值,保持合法 JSON。
@@ -435,6 +454,10 @@ func renderArtifactRef(a *core.ArtifactRef, resolve ArtifactResolver, limit int,
if a == nil {
return ""
}
+ name := a.Name
+ if isImageArtifact(a.Kind, name) {
+ return fmt.Sprintf("此前上传的图片:%s。\n", name)
+ }
if resolve == nil {
return fmt.Sprintf("此前已生成文件:%s。\n", a.Name)
}
@@ -454,6 +477,18 @@ func renderArtifactRef(a *core.ArtifactRef, resolve ArtifactResolver, limit int,
return fmt.Sprintf("此前已生成文件 %s,内容如下:\n%s\n", art.Name, text)
}
+func isImageArtifact(kind, name string) bool {
+ if kind == "image" {
+ return true
+ }
+ switch strings.ToLower(filepath.Ext(name)) {
+ case ".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".svg":
+ return true
+ default:
+ return false
+ }
+}
+
// extractiveSummary 把窗口外历史压成短摘要(不调用模型)。summarizeBeyond 打开时注入。
func extractiveSummary(dropped []core.Message) string {
var lines []string
diff --git a/internal/ctxasm/ctxasm_test.go b/internal/ctxasm/ctxasm_test.go
index 0bb553b..8be4b36 100644
--- a/internal/ctxasm/ctxasm_test.go
+++ b/internal/ctxasm/ctxasm_test.go
@@ -203,3 +203,47 @@ func contains(items []string, want string) bool {
}
return false
}
+
+func TestCurrentImageAttachmentAndHistoryPlaceholder(t *testing.T) {
+ png := []byte{
+ 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52,
+ 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x02, 0x00, 0x00, 0x00, 0x90, 0x77, 0x53,
+ 0xde, 0x00, 0x00, 0x00, 0x0c, 0x49, 0x44, 0x41, 0x54, 0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0x50,
+ 0x0f, 0x00, 0x04, 0x85, 0x01, 0x80, 0x84, 0xa9, 0x8c, 0x21, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45,
+ 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82,
+ }
+ doc := core.ParsedDocument{
+ Meta: core.DocMeta{Filename: "chart.png", MIME: "image/png"},
+ Blocks: []core.DocBlock{{
+ Type: core.BlockImageRef,
+ Ref: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==",
+ }},
+ }
+ msg := BuildCurrentUser(core.EffectiveSettings{}, fakeUser("看图"), []core.ParsedDocument{doc}, nil, &Report{})
+ if len(msg.Images) != 1 || msg.Images[0].MIME != "image/png" || msg.Images[0].Data == "" {
+ t.Fatalf("images %+v", msg.Images)
+ }
+ if !strings.Contains(msg.Content, "图像已随本消息发送") || strings.Contains(msg.Content, string(png)) {
+ t.Fatalf("content %q", msg.Content)
+ }
+
+ prev := core.Message{ID: "old", Role: core.RoleUser, Parts: []core.Part{
+ {Type: core.PartText, Text: "上一张"},
+ {Type: core.PartArtifactRef, Artifact: &core.ArtifactRef{ArtifactID: "a1", Name: "chart.png", Kind: "image"}},
+ }}
+ msgs, _ := AssembleInput(core.AgentProfile{}, core.EffectiveSettings{}, []core.Message{prev}, fakeUser("继续"), nil, nil, nil, nil, nil, func(core.ID) (core.Artifact, []byte, bool) {
+ return core.Artifact{Name: "chart.png", MIME: "image/png"}, png, true
+ }, AssembleOptions{})
+ var hist string
+ for _, m := range msgs {
+ if strings.Contains(m.Content, "上一张") {
+ hist = m.Content
+ if len(m.Images) != 0 {
+ t.Fatal("history must not resend image bytes")
+ }
+ }
+ }
+ if !strings.Contains(hist, "此前上传的图片:chart.png") {
+ t.Fatalf("history placeholder missing: %q", hist)
+ }
+}
diff --git a/internal/ingest/image_test.go b/internal/ingest/image_test.go
new file mode 100644
index 0000000..7a352c7
--- /dev/null
+++ b/internal/ingest/image_test.go
@@ -0,0 +1,28 @@
+package ingest
+
+import (
+ "strings"
+ "testing"
+)
+
+func TestParsePNGBecomesImageBlock(t *testing.T) {
+ doc := Parse("chart.png", "image/png", onePixelPNG)
+ if doc.Meta.Parser != "builtin:image" || len(doc.Blocks) != 1 || doc.Blocks[0].Ref == "" {
+ t.Fatalf("%+v warnings=%v", doc.Meta, doc.Warnings)
+ }
+ if len(doc.Warnings) == 0 || !strings.Contains(doc.Warnings[0], "按图像发送") {
+ t.Fatalf("warnings %v", doc.Warnings)
+ }
+ svg := Parse("logo.svg", "image/svg+xml", []byte(""))
+ if len(svg.Blocks) != 0 || len(svg.Warnings) == 0 || strings.Contains(svg.Warnings[0], "不走文本解析") {
+ t.Fatalf("svg blocks=%d warnings=%v", len(svg.Blocks), svg.Warnings)
+ }
+}
+
+var onePixelPNG = []byte{
+ 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52,
+ 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x02, 0x00, 0x00, 0x00, 0x90, 0x77, 0x53,
+ 0xde, 0x00, 0x00, 0x00, 0x0c, 0x49, 0x44, 0x41, 0x54, 0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0x50,
+ 0x0f, 0x00, 0x04, 0x85, 0x01, 0x80, 0x84, 0xa9, 0x8c, 0x21, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45,
+ 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82,
+}
diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go
index e409c49..78a95a9 100644
--- a/internal/ingest/ingest.go
+++ b/internal/ingest/ingest.go
@@ -12,6 +12,7 @@ import (
"strings"
"time"
+ "agent-system-backend/internal/vision"
core "agent-system-core"
pdflib "github.com/ledongthuc/pdf"
)
@@ -87,7 +88,15 @@ func Parse(filename, mime string, content []byte) core.ParsedDocument {
doc.Warnings = append(doc.Warnings, "旧版二进制格式 ."+ext+" 不解析:请转存为 ."+legacyOfficeTarget(ext)+" 后重传")
default:
if isImage(ext) {
- doc.Warnings = append(doc.Warnings, "图片不走文本解析(OCR 为 MinerU 深度档,远期)")
+ block, warn, ok := vision.PrepareImage(filename, mime, content)
+ if warn != "" {
+ doc.Warnings = append(doc.Warnings, warn)
+ }
+ if ok {
+ doc.Meta.Parser = "builtin:image"
+ doc.Blocks = []core.DocBlock{block}
+ doc.Meta.Confidence = 1
+ }
} else {
// 未知类型按纯文本尝试(二进制内容会产生乱码 warning)。
text := string(content)
@@ -440,6 +449,7 @@ func ParsersDescription() []core.ParserDesc {
{ID: "builtin:xlsx", Formats: []string{"xlsx", "xlsm"}, Source: core.SourceBuiltin},
{ID: "builtin:pptx", Formats: []string{"pptx", "pptm", "ppsx"}, Source: core.SourceBuiltin},
{ID: "builtin:legacy-office", Formats: []string{"doc", "xls", "ppt"}, Source: core.SourceBuiltin, Note: "不解析;提示转存现代格式后重传"},
+ {ID: "builtin:image", Formats: []string{"png", "jpg", "jpeg", "gif", "webp"}, Source: core.SourceBuiltin, Note: "随当前轮消息以图像发送;历史轮只留文字占位"},
{ID: "mineru", Formats: []string{"pdf", "png", "jpg"}, Source: core.SourcePlugin, Note: "远期可选:深度解析与交叉验证(默认不部署)"},
}
}
diff --git a/internal/profiles/profiles.go b/internal/profiles/profiles.go
index bfab9ac..a841df0 100644
--- a/internal/profiles/profiles.go
+++ b/internal/profiles/profiles.go
@@ -12,6 +12,9 @@ import (
const OwnerDefault = store.OwnerDefault
const DefaultProfileID = "default"
+const FastProfileID = "fast"
+const ReasoningProfileID = "reasoning"
+const WritingProfileID = "writing"
// defaultSystemPrompt 默认画像不再携带常驻提示词:基座(身份/行为/谨慎/工具/风格)
// 与环境段由提示词库组装(prompts 库 base.*/env.*);文档任务输出要求已技能化
@@ -22,7 +25,7 @@ func DefaultProfile() core.AgentProfile {
return core.AgentProfile{
ID: DefaultProfileID,
Name: "通用助手",
- Description: "内置默认画像:全部工具可用,HTTP 工具执行前确认(T0 基线)。",
+ Description: "内置默认画像:全部工具可用,HTTP 工具执行前确认(T0 基线)。对话模型由系统设置决定。",
Builtin: true,
Source: "builtin",
SystemPrompt: defaultSystemPrompt,
@@ -35,19 +38,57 @@ func DefaultProfile() core.AgentProfile {
}
}
+// BuiltinProfiles 是启动时幂等写入的全部内置画像。模型名是网关模型 ID,
+// 端点与密钥沿用系统设置。轻量档留空则回落设置里的轻量模型。
+func BuiltinProfiles() []core.AgentProfile {
+ fast := DefaultProfile()
+ fast.ID = FastProfileID
+ fast.Name = "快速助手"
+ fast.Description = "内置画像:对话和轻量任务都用 qwen3.8-flash,适合短问答和改写。"
+ fast.Model = core.ModelRoute{Chat: "qwen3.8-flash", Lightweight: "qwen3.8-flash"}
+
+ reasoning := DefaultProfile()
+ reasoning.ID = ReasoningProfileID
+ reasoning.Name = "深度推理"
+ reasoning.Description = "内置画像:对话用 ZHIPU/GLM-5.3-FlashX。分诊和校验仍走系统轻量模型。"
+ reasoning.Model = core.ModelRoute{Chat: "ZHIPU/GLM-5.3-FlashX"}
+
+ writing := DefaultProfile()
+ writing.ID = WritingProfileID
+ writing.Name = "写作与文档"
+ writing.Description = "内置画像:对话用 ZHIPU/GLM-5.3-Flash。分诊和校验仍走系统轻量模型。"
+ writing.Model = core.ModelRoute{Chat: "ZHIPU/GLM-5.3-Flash"}
+
+ return []core.AgentProfile{DefaultProfile(), fast, reasoning, writing}
+}
+
+func builtinID(id string) bool {
+ for _, p := range BuiltinProfiles() {
+ if p.ID == id {
+ return true
+ }
+ }
+ return false
+}
+
type Service struct{ st *store.Store }
func New(st *store.Store) *Service { return &Service{st: st} }
-// EnsureDefault 幂等写入内置画像;内置画像只读(Save 拒绝编辑),库中与内置定义
+// EnsureDefault 幂等写入全部内置画像;内置画像只读(Save 拒绝编辑),库中与内置定义
// 有差异的行只可能来自更早版本的内置定义,因此差异时升级为当前内置定义。
func (s *Service) EnsureDefault() error {
- if cur, err := s.st.GetProfile(OwnerDefault, DefaultProfileID); err == nil {
- if reflect.DeepEqual(cur, DefaultProfile()) {
- return nil
+ for _, p := range BuiltinProfiles() {
+ if cur, err := s.st.GetProfile(OwnerDefault, p.ID); err == nil {
+ if reflect.DeepEqual(cur, p) {
+ continue
+ }
+ }
+ if err := s.st.SaveProfile(p); err != nil {
+ return err
}
}
- return s.st.SaveProfile(DefaultProfile())
+ return nil
}
func (s *Service) List(owner string) ([]core.AgentProfile, error) { return s.st.ListProfiles(owner) }
@@ -62,7 +103,7 @@ func (s *Service) Get(owner, id string) (core.AgentProfile, error) {
}
func (s *Service) Save(owner string, p core.AgentProfile) error {
- if p.ID == DefaultProfileID {
+ if builtinID(p.ID) {
return errReadOnly("profiles: builtin profile is read-only; duplicate and modify")
}
if p.Tools.Allow == nil {
@@ -72,7 +113,12 @@ func (s *Service) Save(owner string, p core.AgentProfile) error {
return s.st.SaveProfile(p)
}
-func (s *Service) Delete(owner, id string) error { return s.st.DeleteProfile(owner, id) }
+func (s *Service) Delete(owner, id string) error {
+ if builtinID(id) {
+ return errReadOnly("profiles: builtin profile is read-only")
+ }
+ return s.st.DeleteProfile(owner, id)
+}
// SaveAccountChecked is the user-facing write path. Builtin profiles remain
// read-only and the Store binds the write to the actor's effective account.
diff --git a/internal/profiles/profiles_test.go b/internal/profiles/profiles_test.go
new file mode 100644
index 0000000..a4ae387
--- /dev/null
+++ b/internal/profiles/profiles_test.go
@@ -0,0 +1,52 @@
+package profiles
+
+import (
+ "path/filepath"
+ "testing"
+
+ "agent-system-backend/internal/store"
+ core "agent-system-core"
+)
+
+func TestBuiltinProfilesRoundTripAndReadOnly(t *testing.T) {
+ st, err := store.OpenSQLite(filepath.Join(t.TempDir(), "p.db"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { st.Close() })
+ svc := New(st)
+ if err := svc.EnsureDefault(); err != nil {
+ t.Fatal(err)
+ }
+ if err := svc.EnsureDefault(); err != nil {
+ t.Fatal(err)
+ }
+ want := map[string]string{
+ DefaultProfileID: "",
+ FastProfileID: "qwen3.8-flash",
+ ReasoningProfileID: "ZHIPU/GLM-5.3-FlashX",
+ WritingProfileID: "ZHIPU/GLM-5.3-Flash",
+ }
+ for id, model := range want {
+ got, err := svc.Get(OwnerDefault, id)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !got.Builtin || got.Model.Chat != model {
+ t.Fatalf("%s: builtin=%v model=%q", id, got.Builtin, got.Model.Chat)
+ }
+ if err := svc.Save("local", core.AgentProfile{ID: id, Name: "改"}); err == nil {
+ t.Fatalf("save %s should be rejected", id)
+ }
+ if err := svc.Delete("local", id); err == nil {
+ t.Fatalf("delete %s should be rejected", id)
+ }
+ }
+ listed, err := svc.List(OwnerDefault)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(listed) != len(want) {
+ t.Fatalf("listed %d", len(listed))
+ }
+}
diff --git a/internal/runtime/llm.go b/internal/runtime/llm.go
index d06dae1..ca50c7e 100644
--- a/internal/runtime/llm.go
+++ b/internal/runtime/llm.go
@@ -17,6 +17,7 @@ import (
"agent-system-backend/internal/ctxasm"
"agent-system-backend/internal/prompts"
"agent-system-backend/internal/quality"
+ "agent-system-backend/internal/vision"
core "agent-system-core"
)
@@ -228,7 +229,8 @@ func weekdayCN(t time.Time) string {
func (l *LLMRunner) Execute(rc RunCtx) error {
eff := rc.Settings()
prof := rc.Profile()
- route := ResolveChatModel(prof, eff)
+ hasImages := vision.HasVisualInput(rc.Attachments(), currentUserText(rc))
+ route := ResolveVisionModel(prof, eff, hasImages)
if route.Model == "" || route.Base == "" {
return fmt.Errorf("llm: 主模型未配置(当前解析:model=%q base=%q;检查 settings.models.chat 与画像 ModelRoute;演示模式可开 runtime.mock)", route.Model, route.Base)
}
@@ -275,6 +277,7 @@ func (l *LLMRunner) Execute(rc RunCtx) error {
Surfaces: rc.Surfaces(),
ThinkReplay: thinkReplayEnabled(eff, route.Model),
})
+ vision.AttachLinkedImages(rc.Context(), msgs)
tail := ctxasm.Tail{
Instructions: eff.GetString("context.userInstructions", ""),
History: hist,
diff --git a/internal/runtime/llm_direct.go b/internal/runtime/llm_direct.go
index aa40273..c929ee0 100644
--- a/internal/runtime/llm_direct.go
+++ b/internal/runtime/llm_direct.go
@@ -126,6 +126,15 @@ func (t directTool) Execute(ctx context.Context, args json.RawMessage) (agenttoo
return t.run(ctx, args), nil
}
+func messagesHaveImages(msgs []agentllm.Message) bool {
+ for _, m := range msgs {
+ if len(m.Images) > 0 {
+ return true
+ }
+ }
+ return false
+}
+
func (l *LLMRunner) runDirect(rc RunCtx, stream agentllm.StreamFunc, model string, msgs []agentllm.Message, decls []agentllm.ToolDecl, streamOpts agentllm.StreamOptions, cardflow bool, eff core.EffectiveSettings, maxTurns, maxTools int, refresh turnRefresh) error {
st := &directRun{l: l, rc: rc, eff: eff, cardflow: cardflow, started: l.now(), inflight: map[string]*toolSlot{}}
// 记忆端口(N3 WP5):runCtx 暴露 store 时接线(注入由循环按请求完成)。
@@ -161,7 +170,7 @@ func (l *LLMRunner) runDirect(rc RunCtx, stream agentllm.StreamFunc, model strin
break
}
}
- stream = agentllm.WithFallback(stream, ChatFallbacks(eff), st.noteModelSwitch)
+ stream = agentllm.WithFallback(stream, ChatFallbacksFor(st.eff, messagesHaveImages(msgs)), st.noteModelSwitch)
defer func() {
if sink, ok := rc.(interface{ AddUsage(core.Usage) }); ok {
st.switchMu.Lock()
diff --git a/internal/runtime/model.go b/internal/runtime/model.go
index 542ec84..52ffd71 100644
--- a/internal/runtime/model.go
+++ b/internal/runtime/model.go
@@ -3,6 +3,7 @@ package runtime
import (
"strings"
+ "agent-system-backend/internal/vision"
core "agent-system-core"
)
@@ -61,12 +62,48 @@ func ResolveChatModel(prof core.AgentProfile, eff core.EffectiveSettings) Resolv
Key: overlayKey(prof.Owner, "models.chat", eff.GetString("models.chat.apiKey", ""))}
}
-// ChatFallbacks 解析 models.chat.fallbacks。默认空,不改变现有模型选择。
-func ChatFallbacks(eff core.EffectiveSettings) []string {
- raw := strings.TrimSpace(eff.GetString("models.chat.fallbacks", ""))
- if raw == "" {
- return nil
+// ResolveVisionModel 在当前轮含图片时核对视觉清单。清单内的模型保持原选择;
+// 不在清单内则这一轮改用 models.vision.model,端点和密钥仍沿用主模型。
+// 没有图片时与 ResolveChatModel 相同。
+func ResolveVisionModel(prof core.AgentProfile, eff core.EffectiveSettings, hasImages bool) ResolvedModel {
+ chat := ResolveChatModel(prof, eff)
+ if !hasImages || modelCapable(chat.Model, VisionCapable(eff)) {
+ return chat
}
+ name := strings.TrimSpace(eff.GetString("models.vision.model", ""))
+ if name == "" {
+ name = vision.DefaultVisionModel
+ }
+ if name == chat.Model {
+ return chat
+ }
+ chat.Model = name
+ return chat
+}
+
+// VisionCapable 解析 models.vision.capable。空字符串用内置默认清单。
+func VisionCapable(eff core.EffectiveSettings) []string {
+ raw := strings.TrimSpace(eff.GetString("models.vision.capable", ""))
+ if raw == "" {
+ raw = vision.DefaultVisionCapable
+ }
+ return splitModelList(raw)
+}
+
+func modelCapable(name string, capable []string) bool {
+ name = strings.TrimSpace(name)
+ if name == "" {
+ return false
+ }
+ for _, item := range capable {
+ if item == name {
+ return true
+ }
+ }
+ return false
+}
+
+func splitModelList(raw string) []string {
var out []string
seen := map[string]bool{}
for _, part := range strings.Split(raw, ",") {
@@ -80,6 +117,27 @@ func ChatFallbacks(eff core.EffectiveSettings) []string {
return out
}
+// ChatFallbacks 解析 models.chat.fallbacks。默认空,不改变现有模型选择。
+func ChatFallbacks(eff core.EffectiveSettings) []string {
+ return splitModelList(eff.GetString("models.chat.fallbacks", ""))
+}
+
+// ChatFallbacksFor 在图片轮只保留视觉清单内的回退模型,避免退到不能看图的模型。
+func ChatFallbacksFor(eff core.EffectiveSettings, hasImages bool) []string {
+ all := ChatFallbacks(eff)
+ if !hasImages {
+ return all
+ }
+ capable := VisionCapable(eff)
+ var out []string
+ for _, name := range all {
+ if modelCapable(name, capable) {
+ out = append(out, name)
+ }
+ }
+ return out
+}
+
// ResolveLightweightModel 解析轻量模型:画像 route > 设置层轻量档 > 回落主档。
func ResolveLightweightModel(prof core.AgentProfile, eff core.EffectiveSettings) ResolvedModel {
m := eff.GetString("models.lightweight.model", "")
diff --git a/internal/runtime/model_test.go b/internal/runtime/model_test.go
index bac984b..ec93d99 100644
--- a/internal/runtime/model_test.go
+++ b/internal/runtime/model_test.go
@@ -58,3 +58,35 @@ func TestChatFallbacksDefaultEmpty(t *testing.T) {
t.Fatalf("%#v", got)
}
}
+
+func TestResolveVisionModel(t *testing.T) {
+ eff := effOf(map[string]any{"models": map[string]any{"chat": map[string]any{
+ "model": "deepseek-v4.1-flash", "baseUrl": "https://gw.example/v1", "apiKey": "k",
+ "fallbacks": "plain-text,ZHIPU/GLM-5.3-FlashX",
+ }, "vision": map[string]any{
+ "model": "qwen3.8-flash",
+ "capable": "deepseek-v4.1-flash,qwen3.8-flash,ZHIPU/GLM-5.3-FlashX",
+ }}})
+ prof := core.AgentProfile{}
+ got := ResolveVisionModel(prof, eff, false)
+ if got.Model != "deepseek-v4.1-flash" {
+ t.Fatalf("no image should keep chat model: %+v", got)
+ }
+ got = ResolveVisionModel(prof, eff, true)
+ if got.Model != "deepseek-v4.1-flash" || got.Base != "https://gw.example/v1" || got.Key != "k" {
+ t.Fatalf("capable model should stay: %+v", got)
+ }
+ prof.Model.Chat = "plain-text"
+ got = ResolveVisionModel(prof, eff, true)
+ if got.Model != "qwen3.8-flash" || got.Base != "https://gw.example/v1" || got.Key != "k" {
+ t.Fatalf("incapable model should swap, keeping endpoint: %+v", got)
+ }
+ fb := ChatFallbacksFor(eff, true)
+ if len(fb) != 1 || fb[0] != "ZHIPU/GLM-5.3-FlashX" {
+ t.Fatalf("image fallbacks %#v", fb)
+ }
+ fb = ChatFallbacksFor(eff, false)
+ if len(fb) != 2 || fb[0] != "plain-text" {
+ t.Fatalf("text fallbacks %#v", fb)
+ }
+}
diff --git a/internal/settings/settings.go b/internal/settings/settings.go
index b6ea8a4..d2f5c21 100644
--- a/internal/settings/settings.go
+++ b/internal/settings/settings.go
@@ -39,8 +39,12 @@ func SystemDefaults() map[string]any {
"fallbacks": "", // 逗号分隔;空 = 不回退
},
"lightweight": map[string]any{"baseUrl": "", "apiKey": "", "model": ""},
- "embedding": map[string]any{"baseUrl": "", "apiKey": "", "model": ""},
- "rerank": map[string]any{"baseUrl": "", "apiKey": "", "model": ""},
+ "vision": map[string]any{
+ "model": "qwen3.8-flash",
+ "capable": "deepseek-v4.1-flash,qwen3.8-flash,ZHIPU/GLM-5.3-FlashX,ZHIPU/GLM-5.3-Flash",
+ },
+ "embedding": map[string]any{"baseUrl": "", "apiKey": "", "model": ""},
+ "rerank": map[string]any{"baseUrl": "", "apiKey": "", "model": ""},
},
"tools": map[string]any{
"tavily": map[string]any{"apiKeys": "", "endpoint": ""}, // 网络搜索(逗号分隔密钥池;空 = 不注册搜索工具)
@@ -100,6 +104,8 @@ func Schema() []core.SettingDesc {
{Key: "models.lightweight.baseUrl", Group: "models", Label: "轻量模型 Base URL", Type: core.SettingString, Default: "", Effect: core.EffectHot, Scope: core.SettingScopeSystem, Description: "标题生成/澄清等轻任务;空则回落主模型"},
{Key: "models.lightweight.apiKey", Group: "models", Label: "轻量模型 API Key", Type: core.SettingSecret, Default: "", Effect: core.EffectHot, Scope: core.SettingScopeSystem},
{Key: "models.lightweight.model", Group: "models", Label: "轻量模型名称", Type: core.SettingString, Default: "", Effect: core.EffectNewRun, Scope: core.SettingScopeSystem},
+ {Key: "models.vision.model", Group: "models", Label: "视觉兜底模型", Type: core.SettingString, Default: "qwen3.8-flash", Effect: core.EffectNewRun, Scope: core.SettingScopeSystem, Description: "当前轮含图片且主模型不在视觉清单内时改用此模型。端点与密钥沿用主模型"},
+ {Key: "models.vision.capable", Group: "models", Label: "可看图模型", Type: core.SettingString, Default: "deepseek-v4.1-flash,qwen3.8-flash,ZHIPU/GLM-5.3-FlashX,ZHIPU/GLM-5.3-Flash", Effect: core.EffectNewRun, Scope: core.SettingScopeSystem, Description: "逗号分隔。当前轮有图片时,主模型和回退链只保留清单内的名字。空则使用内置默认清单"},
{Key: "models.lightweight.maxOutputTokens", Group: "models", Label: "轻量模型输出上限", Type: core.SettingNumber, Default: 0, Effect: core.EffectNewRun, Scope: core.SettingScopeSystem, Description: "单次回复的 max_tokens。0 = 不限制,由模型服务端决定"},
{Key: "models.embedding.baseUrl", Group: "models", Label: "Embedding Base URL", Type: core.SettingString, Default: "", Effect: core.EffectHot, Scope: core.SettingScopeSystem, Description: "RAG/记忆检索用(向量管线远期启用)"},
{Key: "models.embedding.apiKey", Group: "models", Label: "Embedding API Key", Type: core.SettingSecret, Default: "", Effect: core.EffectHot, Scope: core.SettingScopeSystem},
diff --git a/internal/vision/fetch.go b/internal/vision/fetch.go
new file mode 100644
index 0000000..085afac
--- /dev/null
+++ b/internal/vision/fetch.go
@@ -0,0 +1,193 @@
+package vision
+
+import (
+ "context"
+ "encoding/base64"
+ "errors"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "net/url"
+ "strings"
+ "time"
+
+ agentllm "agent-native/runtime/llm"
+)
+
+var (
+ errPrivateAddress = errors.New("地址不在允许范围")
+ errNotImage = errors.New("响应不是受支持的图片")
+ errTooLarge = errors.New("超过单图大小上限")
+)
+
+// FetchImage 下载一张公网图片。拒绝回环、私网、链路本地和云元数据地址,
+// 并把连接钉在解析到的公网 IP 上,避免 DNS 重绑定。不走环境变量里的代理。
+func FetchImage(ctx context.Context, rawURL string) (agentllm.Image, error) {
+ if err := validateImageURL(ctx, rawURL); err != nil {
+ return agentllm.Image{}, err
+ }
+ client := &http.Client{
+ Timeout: 20 * time.Second,
+ CheckRedirect: func(req *http.Request, via []*http.Request) error {
+ if len(via) >= 3 {
+ return errors.New("重定向过多")
+ }
+ if err := validateImageURL(req.Context(), req.URL.String()); err != nil {
+ return err
+ }
+ return nil
+ },
+ Transport: &http.Transport{
+ Proxy: nil,
+ DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+ host, port, err := net.SplitHostPort(addr)
+ if err != nil {
+ return nil, err
+ }
+ ip, err := pickPublicIP(ctx, host)
+ if err != nil {
+ return nil, err
+ }
+ var d net.Dialer
+ d.Timeout = 10 * time.Second
+ return d.DialContext(ctx, network, net.JoinHostPort(ip.String(), port))
+ },
+ ResponseHeaderTimeout: 10 * time.Second,
+ TLSHandshakeTimeout: 10 * time.Second,
+ },
+ }
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
+ if err != nil {
+ return agentllm.Image{}, err
+ }
+ req.Header.Set("User-Agent", "agent-system-vision/1.0")
+ resp, err := client.Do(req)
+ if err != nil {
+ if errors.Is(err, errPrivateAddress) {
+ return agentllm.Image{}, errPrivateAddress
+ }
+ return agentllm.Image{}, fmt.Errorf("下载失败")
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ return agentllm.Image{}, fmt.Errorf("下载失败")
+ }
+ body, err := io.ReadAll(io.LimitReader(resp.Body, MaxImageBytes+1))
+ if err != nil {
+ return agentllm.Image{}, fmt.Errorf("下载失败")
+ }
+ if len(body) > MaxImageBytes {
+ return agentllm.Image{}, errTooLarge
+ }
+ kind := MIMEOf(body)
+ if kind == "" || !contentTypeOK(resp.Header.Get("Content-Type"), kind) {
+ return agentllm.Image{}, errNotImage
+ }
+ // 只回传字节。URL 留给网关自己去拉会绕过这里的地址校验。
+ return agentllm.Image{
+ MIME: kind,
+ Data: base64.StdEncoding.EncodeToString(body),
+ }, nil
+}
+
+func publicReason(err error) string {
+ switch {
+ case err == nil:
+ return ""
+ case errors.Is(err, errPrivateAddress):
+ return errPrivateAddress.Error()
+ case errors.Is(err, errNotImage):
+ return errNotImage.Error()
+ case errors.Is(err, errTooLarge):
+ return errTooLarge.Error()
+ default:
+ return "下载失败"
+ }
+}
+
+func validateImageURL(ctx context.Context, raw string) error {
+ u, err := url.Parse(raw)
+ if err != nil || u.Host == "" {
+ return errPrivateAddress
+ }
+ if u.User != nil {
+ return errPrivateAddress
+ }
+ switch strings.ToLower(u.Scheme) {
+ case "http", "https":
+ default:
+ return errPrivateAddress
+ }
+ if _, err := pickPublicIP(ctx, u.Hostname()); err != nil {
+ return errPrivateAddress
+ }
+ return nil
+}
+
+func pickPublicIP(ctx context.Context, host string) (net.IP, error) {
+ if ip := net.ParseIP(host); ip != nil {
+ if err := checkIP(ip); err != nil {
+ return nil, err
+ }
+ return ip, nil
+ }
+ if host == "" || strings.EqualFold(host, "localhost") {
+ return nil, errPrivateAddress
+ }
+ addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
+ if err != nil || len(addrs) == 0 {
+ return nil, errPrivateAddress
+ }
+ var first net.IP
+ for _, addr := range addrs {
+ if err := checkIP(addr.IP); err != nil {
+ return nil, err
+ }
+ if first == nil {
+ first = addr.IP
+ }
+ }
+ if first == nil {
+ return nil, errPrivateAddress
+ }
+ return first, nil
+}
+
+var (
+ cgnat = mustCIDR("100.64.0.0/10")
+)
+
+func mustCIDR(s string) *net.IPNet {
+ _, n, err := net.ParseCIDR(s)
+ if err != nil {
+ panic(err)
+ }
+ return n
+}
+
+func checkIP(ip net.IP) error {
+ if ip == nil {
+ return errPrivateAddress
+ }
+ if v4 := ip.To4(); v4 != nil {
+ ip = v4
+ }
+ switch {
+ case ip.IsLoopback(), ip.IsPrivate(), ip.IsLinkLocalUnicast(), ip.IsLinkLocalMulticast(),
+ ip.IsMulticast(), ip.IsUnspecified(), cgnat.Contains(ip):
+ return errPrivateAddress
+ default:
+ return nil
+ }
+}
+
+func contentTypeOK(header, sniffed string) bool {
+ ct := strings.ToLower(strings.TrimSpace(strings.Split(header, ";")[0]))
+ switch ct {
+ case "", "application/octet-stream", "binary/octet-stream":
+ return sniffed != ""
+ default:
+ return ct == sniffed
+ }
+}
diff --git a/internal/vision/images.go b/internal/vision/images.go
new file mode 100644
index 0000000..dba0ed6
--- /dev/null
+++ b/internal/vision/images.go
@@ -0,0 +1,136 @@
+package vision
+
+import (
+ "encoding/base64"
+ "fmt"
+ "path/filepath"
+ "strings"
+
+ agentllm "agent-native/runtime/llm"
+ core "agent-system-core"
+)
+
+// PrepareImage 把上传的图片收成一条 image-ref。Ref 是标准 base64(无换行)。
+// 不支持的类型和超限图片返回警告且 ok 为 false,调用方必须把警告展示出来。
+func PrepareImage(filename, mime string, content []byte) (core.DocBlock, string, bool) {
+ ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(filename), "."))
+ kind, known := sendableExt[ext]
+ if !known {
+ return core.DocBlock{}, fmt.Sprintf("图片 .%s 不作为图像发送(请转为 png、jpeg、gif 或 webp)", ext), false
+ }
+ if len(content) == 0 {
+ return core.DocBlock{}, "图片内容为空,本轮不发送图像", false
+ }
+ if len(content) > MaxImageBytes {
+ return core.DocBlock{}, fmt.Sprintf("图片超过 %dMB,本轮不发送图像(请压缩后重传)", MaxImageBytes>>20), false
+ }
+ detected := sniffImage(content)
+ if detected == "" || detected != kind {
+ return core.DocBlock{}, "图片内容与扩展名不符,本轮不发送图像", false
+ }
+ if mime != "" && !strings.EqualFold(strings.TrimSpace(strings.Split(mime, ";")[0]), kind) && !strings.EqualFold(mime, "application/octet-stream") {
+ return core.DocBlock{}, "图片类型与内容不符,本轮不发送图像", false
+ }
+ return core.DocBlock{
+ Type: core.BlockImageRef,
+ Ref: base64.StdEncoding.EncodeToString(content),
+ Text: filename,
+ }, "图片按图像发送给模型,不转成文字", true
+}
+
+// ImagesFromDoc 取出文档里已编码的图片。MIME 来自文件名,不信任外部声明。
+func ImagesFromDoc(doc core.ParsedDocument) []agentllm.Image {
+ ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(doc.Meta.Filename), "."))
+ kind := sendableExt[ext]
+ if kind == "" {
+ return nil
+ }
+ var out []agentllm.Image
+ for _, b := range doc.Blocks {
+ if b.Type != core.BlockImageRef || strings.TrimSpace(b.Ref) == "" {
+ continue
+ }
+ out = append(out, agentllm.Image{MIME: kind, Data: b.Ref})
+ }
+ return out
+}
+
+// HasVisualInput 判断这一轮是否带图:已解析的图片附件,或正文里的图片链接。
+func HasVisualInput(docs []core.ParsedDocument, text string) bool {
+ for _, doc := range docs {
+ if len(ImagesFromDoc(doc)) > 0 {
+ return true
+ }
+ }
+ return len(ImageLinks(text)) > 0
+}
+
+// Admit 按张数和总字节数收下一批图片。被拒绝的每一项都有一句说明。
+func Admit(existing []agentllm.Image, incoming []agentllm.Image) (kept []agentllm.Image, notes []string) {
+ count, total := usage(existing)
+ for _, img := range incoming {
+ n := decodedLen(img.Data)
+ switch {
+ case n <= 0:
+ notes = append(notes, "一张图片没有可用内容,已跳过")
+ case n > MaxImageBytes:
+ notes = append(notes, fmt.Sprintf("一张图片超过 %dMB,已跳过", MaxImageBytes>>20))
+ case count >= MaxImages:
+ notes = append(notes, fmt.Sprintf("本轮图片超过 %d 张,其余未发送", MaxImages))
+ return kept, notes
+ case total+n > MaxTotalBytes:
+ notes = append(notes, fmt.Sprintf("本轮图片合计超过 %dMB,其余未发送", MaxTotalBytes>>20))
+ return kept, notes
+ default:
+ kept = append(kept, img)
+ count++
+ total += n
+ }
+ }
+ return kept, notes
+}
+
+func usage(images []agentllm.Image) (count, total int) {
+ for _, img := range images {
+ n := decodedLen(img.Data)
+ if n > 0 {
+ count++
+ total += n
+ }
+ }
+ return count, total
+}
+
+func decodedLen(data string) int {
+ raw, err := base64.StdEncoding.DecodeString(data)
+ if err != nil {
+ return 0
+ }
+ return len(raw)
+}
+
+var sendableExt = map[string]string{
+ "png": "image/png",
+ "jpg": "image/jpeg",
+ "jpeg": "image/jpeg",
+ "gif": "image/gif",
+ "webp": "image/webp",
+}
+
+func sniffImage(b []byte) string {
+ switch {
+ case len(b) >= 8 && string(b[:8]) == "\x89PNG\r\n\x1a\n":
+ return "image/png"
+ case len(b) >= 3 && b[0] == 0xFF && b[1] == 0xD8 && b[2] == 0xFF:
+ return "image/jpeg"
+ case len(b) >= 6 && (string(b[:6]) == "GIF87a" || string(b[:6]) == "GIF89a"):
+ return "image/gif"
+ case len(b) >= 12 && string(b[:4]) == "RIFF" && string(b[8:12]) == "WEBP":
+ return "image/webp"
+ default:
+ return ""
+ }
+}
+
+// MIMEOf 返回嗅探结果,供下载后的响应校验。
+func MIMEOf(content []byte) string { return sniffImage(content) }
diff --git a/internal/vision/links.go b/internal/vision/links.go
new file mode 100644
index 0000000..72f2c71
--- /dev/null
+++ b/internal/vision/links.go
@@ -0,0 +1,90 @@
+package vision
+
+import (
+ "context"
+ "fmt"
+ "regexp"
+ "strings"
+
+ agentllm "agent-native/runtime/llm"
+)
+
+// 只认路径以图片扩展名结尾的 http(s) 链接。不根据任意 URL 去探测,
+// 避免把用户消息里的普通网页都拿去下载。
+var imageURLRe = regexp.MustCompile(`(?i)https?://[^\s<>"',。、;:]+\.(?:png|jpe?g|gif|webp)(?:\?[^\s<>"',。、;:]*)?`)
+
+// ImageLinks 按出现顺序返回去重后的图片链接。
+func ImageLinks(text string) []string {
+ seen := map[string]bool{}
+ var out []string
+ for _, raw := range imageURLRe.FindAllString(text, -1) {
+ u := strings.TrimRight(raw, ".,;:)]>\"'")
+ if u == "" || seen[u] {
+ continue
+ }
+ seen[u] = true
+ out = append(out, u)
+ }
+ return out
+}
+
+// AttachLinkedImages 下载最后一条用户消息里的图片链接,并写进该消息的 Images。
+// 下载失败、超限或地址被拒绝时,把原因追加到正文,不静默丢掉。
+func AttachLinkedImages(ctx context.Context, msgs []agentllm.Message) []string {
+ idx := -1
+ for i := len(msgs) - 1; i >= 0; i-- {
+ if msgs[i].Role == agentllm.RoleUser {
+ idx = i
+ break
+ }
+ }
+ if idx < 0 {
+ return nil
+ }
+ links := ImageLinks(msgs[idx].Content)
+ if len(links) == 0 {
+ return nil
+ }
+ var notes []string
+ var incoming []agentllm.Image
+ byData := map[string]string{}
+ for _, link := range links {
+ img, err := FetchImage(ctx, link)
+ if err != nil {
+ notes = append(notes, fmt.Sprintf("%s 未能下载:%s", link, publicReason(err)))
+ continue
+ }
+ incoming = append(incoming, img)
+ byData[img.Data] = link
+ }
+ kept, dropped := Admit(msgs[idx].Images, incoming)
+ msgs[idx].Images = append(msgs[idx].Images, kept...)
+ sent := map[string]bool{}
+ for _, img := range kept {
+ if link := byData[img.Data]; link != "" {
+ notes = append(notes, link+" 已随消息发送")
+ sent[img.Data] = true
+ }
+ }
+ for _, img := range incoming {
+ if !sent[img.Data] {
+ if link := byData[img.Data]; link != "" {
+ notes = append(notes, link+" 未发送")
+ }
+ }
+ }
+ notes = append(notes, dropped...)
+ if len(notes) == 0 {
+ return nil
+ }
+ msgs[idx].Content += "\n\n[图片链接]\n" + strings.Join(prefixLines(notes), "\n") + "\n"
+ return notes
+}
+
+func prefixLines(notes []string) []string {
+ out := make([]string, len(notes))
+ for i, n := range notes {
+ out[i] = "- " + n
+ }
+ return out
+}
diff --git a/internal/vision/models.go b/internal/vision/models.go
new file mode 100644
index 0000000..b0086fc
--- /dev/null
+++ b/internal/vision/models.go
@@ -0,0 +1,18 @@
+// Package vision 负责「这一轮要不要把图发给模型」:能力清单、图片字节预算、
+// 以及用户粘贴的图片链接的受控下载。模型名和端点仍由 runtime 的路由解析。
+package vision
+
+// 默认可看图清单与兜底模型。网关路由变化时改设置 models.vision.capable,
+// 不必改代码。空设置回落到这里。
+const (
+ DefaultVisionModel = "qwen3.8-flash"
+ DefaultVisionCapable = "deepseek-v4.1-flash,qwen3.8-flash,ZHIPU/GLM-5.3-FlashX,ZHIPU/GLM-5.3-Flash"
+)
+
+// 单图与单轮上限。按网关实测的小图请求体留出余量:base64 约膨胀 4/3,
+// 8MB 原图约 11MB 请求体,单轮合计 20MB 原图。
+const (
+ MaxImageBytes = 8 << 20
+ MaxImages = 8
+ MaxTotalBytes = 20 << 20
+)
diff --git a/internal/vision/vision_test.go b/internal/vision/vision_test.go
new file mode 100644
index 0000000..91abee4
--- /dev/null
+++ b/internal/vision/vision_test.go
@@ -0,0 +1,120 @@
+package vision
+
+import (
+ "context"
+ "encoding/base64"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+
+ agentllm "agent-native/runtime/llm"
+ core "agent-system-core"
+)
+
+func TestCheckIP(t *testing.T) {
+ blocked := []string{"127.0.0.1", "10.1.2.3", "192.168.1.1", "172.16.0.1", "169.254.169.254", "100.64.0.1", "::1", "fc00::1", "0.0.0.0"}
+ for _, s := range blocked {
+ if err := checkIP(net.ParseIP(s)); err == nil {
+ t.Errorf("%s should be blocked", s)
+ }
+ }
+ if err := checkIP(net.ParseIP("8.8.8.8")); err != nil {
+ t.Fatal(err)
+ }
+ if err := checkIP(net.ParseIP("1.1.1.1")); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestValidateImageURLRejectsUnsafeTargets(t *testing.T) {
+ ctx := context.Background()
+ for _, raw := range []string{
+ "http://127.0.0.1/a.png",
+ "http://169.254.169.254/latest/a.png",
+ "http://10.0.0.8/a.jpg",
+ "http://user:pass@8.8.8.8/a.png",
+ "file:///tmp/a.png",
+ "http://localhost/a.png",
+ } {
+ if err := validateImageURL(ctx, raw); err == nil {
+ t.Errorf("accepted %s", raw)
+ }
+ }
+ if err := validateImageURL(ctx, "https://8.8.8.8/a.png"); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestFetchImageRejectsLoopbackServer(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "image/png")
+ _, _ = w.Write(onePixelPNG)
+ }))
+ defer srv.Close()
+ _, err := FetchImage(context.Background(), srv.URL+"/pixel.png")
+ if err == nil || !strings.Contains(err.Error(), "允许范围") {
+ t.Fatalf("err = %v", err)
+ }
+}
+
+func TestImageLinksAndAttachNote(t *testing.T) {
+ text := "看这张 http://127.0.0.1/a.png。还有 https://cdn.example.com/b.JPG?x=1 和重复 http://127.0.0.1/a.png"
+ links := ImageLinks(text)
+ if len(links) != 2 || links[0] != "http://127.0.0.1/a.png" || !strings.Contains(links[1], "b.JPG") {
+ t.Fatalf("%#v", links)
+ }
+ msgs := []agentllm.Message{{Role: agentllm.RoleUser, Content: "请看 http://127.0.0.1/secret.png"}}
+ notes := AttachLinkedImages(context.Background(), msgs)
+ if len(notes) == 0 || len(msgs[0].Images) != 0 {
+ t.Fatalf("notes=%v images=%d", notes, len(msgs[0].Images))
+ }
+ if !strings.Contains(msgs[0].Content, "地址不在允许范围") {
+ t.Fatalf("content missing reason: %s", msgs[0].Content)
+ }
+}
+
+func TestPrepareImageAndBudget(t *testing.T) {
+ block, warn, ok := PrepareImage("pixel.png", "image/png", onePixelPNG)
+ if !ok || !strings.Contains(warn, "按图像发送") || block.Type != core.BlockImageRef || block.Ref == "" {
+ t.Fatalf("ok=%v warn=%q block=%+v", ok, warn, block)
+ }
+ if _, warn, ok := PrepareImage("icon.svg", "image/svg+xml", []byte("")); ok || !strings.Contains(warn, "不作为图像发送") {
+ t.Fatalf("svg ok=%v warn=%q", ok, warn)
+ }
+ big := make([]byte, MaxImageBytes+1)
+ if _, warn, ok := PrepareImage("big.png", "image/png", big); ok || !strings.Contains(warn, "超过") {
+ t.Fatalf("big ok=%v warn=%q", ok, warn)
+ }
+ if _, warn, ok := PrepareImage("bad.png", "image/png", []byte("not a png")); ok || !strings.Contains(warn, "不符") {
+ t.Fatalf("bad ok=%v warn=%q", ok, warn)
+ }
+
+ img := agentllm.Image{MIME: "image/png", Data: block.Ref}
+ kept, notes := Admit(nil, []agentllm.Image{img})
+ if len(kept) != 1 || len(notes) != 0 {
+ t.Fatalf("kept=%d notes=%v", len(kept), notes)
+ }
+ many := make([]agentllm.Image, MaxImages+1)
+ for i := range many {
+ many[i] = img
+ }
+ kept, notes = Admit(nil, many)
+ if len(kept) != MaxImages || len(notes) == 0 {
+ t.Fatalf("kept=%d notes=%v", len(kept), notes)
+ }
+ raw, err := base64.StdEncoding.DecodeString(block.Ref)
+ if err != nil || len(raw) != len(onePixelPNG) {
+ t.Fatal(err)
+ }
+}
+
+// 1x1 PNG。
+var onePixelPNG = []byte{
+ 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52,
+ 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x02, 0x00, 0x00, 0x00, 0x90, 0x77, 0x53,
+ 0xde, 0x00, 0x00, 0x00, 0x0c, 0x49, 0x44, 0x41, 0x54, 0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0x50,
+ 0x0f, 0x00, 0x04, 0x85, 0x01, 0x80, 0x84, 0xa9, 0x8c, 0x21, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45,
+ 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82,
+}