Merge pull request 'Send current-turn images and keep vision fallbacks capable' (#203) from feature/four-model-vision into main
ci / vet-and-test (push) Successful in 1m2s

Why: land current-turn image routing on backend main.
Evidence: vet-and-test success on 629c2502ad02e33708fe7ee39afd4b45d1b25483; parent pin 6532fad remains an ancestor.
Impact: vision routing is on backend main.
Refs: #203
This commit was merged in pull request #203.
This commit is contained in:
2026-10-15 06:38:05 +00:00
16 changed files with 899 additions and 19 deletions
+36 -1
View File
@@ -4,12 +4,14 @@ package ctxasm
import (
"encoding/json"
"fmt"
"path/filepath"
"regexp"
"strings"
"time"
"unicode/utf8"
agentllm "agent-native/runtime/llm"
"agent-system-backend/internal/vision"
core "agent-system-core"
)
@@ -134,6 +136,7 @@ func AssembleInput(prof core.AgentProfile, eff core.EffectiveSettings, history [
// 此处与整段组装共用同一实现,保证两条路径的当前轮语义一致。
func BuildCurrentUser(eff core.EffectiveSettings, currentUser core.Message, attachments []core.ParsedDocument, knowledge []core.RetrievalHit, rep *Report) agentllm.Message {
var sb strings.Builder
var images []agentllm.Image
for _, p := range currentUser.Parts {
if p.Type == core.PartText {
sb.WriteString(p.Text)
@@ -146,6 +149,22 @@ func BuildCurrentUser(eff core.EffectiveSettings, currentUser core.Message, atta
}
sb.WriteString("\n\n[附件](用户上传文档的解析全文,按原文顺序保留标题/列表/表格;超长时文末标注截断。请依据文档内容完成任务,输出完整不省略)\n")
for _, doc := range attachments {
if pics := vision.ImagesFromDoc(doc); len(pics) > 0 {
kept, notes := vision.Admit(images, pics)
images = append(images, kept...)
if len(kept) > 0 {
fmt.Fprintf(&sb, "### %s\n[图片 %s,图像已随本消息发送]\n", doc.Meta.Filename, doc.Meta.Filename)
} else {
fmt.Fprintf(&sb, "### %s\n[图片 %s,本轮未发送图像]\n", doc.Meta.Filename, doc.Meta.Filename)
}
for _, note := range notes {
fmt.Fprintf(&sb, "%s\n", note)
if rep != nil {
rep.Truncations = append(rep.Truncations, fmt.Sprintf("附件 %s:%s", doc.Meta.Filename, note))
}
}
continue
}
text := doc.PlainText()
if runes := []rune(text); len(runes) > limit {
text = string(runes[:limit]) + "…(附件超长截断)"
@@ -168,7 +187,7 @@ func BuildCurrentUser(eff core.EffectiveSettings, currentUser core.Message, atta
rep.Items = append(rep.Items, fmt.Sprintf("knowledge(%d)", len(knowledge)))
}
}
return agentllm.Message{Role: agentllm.RoleUser, Content: sb.String()}
return agentllm.Message{Role: agentllm.RoleUser, Content: sb.String(), Images: images}
}
// maxToolArgRunes 是回放进模型的单次工具参数上限。超限时截断字符串值,保持合法 JSON。
@@ -435,6 +454,10 @@ func renderArtifactRef(a *core.ArtifactRef, resolve ArtifactResolver, limit int,
if a == nil {
return ""
}
name := a.Name
if isImageArtifact(a.Kind, name) {
return fmt.Sprintf("此前上传的图片:%s。\n", name)
}
if resolve == nil {
return fmt.Sprintf("此前已生成文件:%s。\n", a.Name)
}
@@ -454,6 +477,18 @@ func renderArtifactRef(a *core.ArtifactRef, resolve ArtifactResolver, limit int,
return fmt.Sprintf("此前已生成文件 %s,内容如下:\n%s\n", art.Name, text)
}
func isImageArtifact(kind, name string) bool {
if kind == "image" {
return true
}
switch strings.ToLower(filepath.Ext(name)) {
case ".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".svg":
return true
default:
return false
}
}
// extractiveSummary 把窗口外历史压成短摘要(不调用模型)。summarizeBeyond 打开时注入。
func extractiveSummary(dropped []core.Message) string {
var lines []string
+44
View File
@@ -203,3 +203,47 @@ func contains(items []string, want string) bool {
}
return false
}
func TestCurrentImageAttachmentAndHistoryPlaceholder(t *testing.T) {
png := []byte{
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52,
0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x02, 0x00, 0x00, 0x00, 0x90, 0x77, 0x53,
0xde, 0x00, 0x00, 0x00, 0x0c, 0x49, 0x44, 0x41, 0x54, 0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0x50,
0x0f, 0x00, 0x04, 0x85, 0x01, 0x80, 0x84, 0xa9, 0x8c, 0x21, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45,
0x4e, 0x44, 0xae, 0x42, 0x60, 0x82,
}
doc := core.ParsedDocument{
Meta: core.DocMeta{Filename: "chart.png", MIME: "image/png"},
Blocks: []core.DocBlock{{
Type: core.BlockImageRef,
Ref: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==",
}},
}
msg := BuildCurrentUser(core.EffectiveSettings{}, fakeUser("看图"), []core.ParsedDocument{doc}, nil, &Report{})
if len(msg.Images) != 1 || msg.Images[0].MIME != "image/png" || msg.Images[0].Data == "" {
t.Fatalf("images %+v", msg.Images)
}
if !strings.Contains(msg.Content, "图像已随本消息发送") || strings.Contains(msg.Content, string(png)) {
t.Fatalf("content %q", msg.Content)
}
prev := core.Message{ID: "old", Role: core.RoleUser, Parts: []core.Part{
{Type: core.PartText, Text: "上一张"},
{Type: core.PartArtifactRef, Artifact: &core.ArtifactRef{ArtifactID: "a1", Name: "chart.png", Kind: "image"}},
}}
msgs, _ := AssembleInput(core.AgentProfile{}, core.EffectiveSettings{}, []core.Message{prev}, fakeUser("继续"), nil, nil, nil, nil, nil, func(core.ID) (core.Artifact, []byte, bool) {
return core.Artifact{Name: "chart.png", MIME: "image/png"}, png, true
}, AssembleOptions{})
var hist string
for _, m := range msgs {
if strings.Contains(m.Content, "上一张") {
hist = m.Content
if len(m.Images) != 0 {
t.Fatal("history must not resend image bytes")
}
}
}
if !strings.Contains(hist, "此前上传的图片:chart.png") {
t.Fatalf("history placeholder missing: %q", hist)
}
}
+28
View File
@@ -0,0 +1,28 @@
package ingest
import (
"strings"
"testing"
)
func TestParsePNGBecomesImageBlock(t *testing.T) {
doc := Parse("chart.png", "image/png", onePixelPNG)
if doc.Meta.Parser != "builtin:image" || len(doc.Blocks) != 1 || doc.Blocks[0].Ref == "" {
t.Fatalf("%+v warnings=%v", doc.Meta, doc.Warnings)
}
if len(doc.Warnings) == 0 || !strings.Contains(doc.Warnings[0], "按图像发送") {
t.Fatalf("warnings %v", doc.Warnings)
}
svg := Parse("logo.svg", "image/svg+xml", []byte("<svg/>"))
if len(svg.Blocks) != 0 || len(svg.Warnings) == 0 || strings.Contains(svg.Warnings[0], "不走文本解析") {
t.Fatalf("svg blocks=%d warnings=%v", len(svg.Blocks), svg.Warnings)
}
}
var onePixelPNG = []byte{
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52,
0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x02, 0x00, 0x00, 0x00, 0x90, 0x77, 0x53,
0xde, 0x00, 0x00, 0x00, 0x0c, 0x49, 0x44, 0x41, 0x54, 0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0x50,
0x0f, 0x00, 0x04, 0x85, 0x01, 0x80, 0x84, 0xa9, 0x8c, 0x21, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45,
0x4e, 0x44, 0xae, 0x42, 0x60, 0x82,
}
+11 -1
View File
@@ -12,6 +12,7 @@ import (
"strings"
"time"
"agent-system-backend/internal/vision"
core "agent-system-core"
pdflib "github.com/ledongthuc/pdf"
)
@@ -87,7 +88,15 @@ func Parse(filename, mime string, content []byte) core.ParsedDocument {
doc.Warnings = append(doc.Warnings, "旧版二进制格式 ."+ext+" 不解析:请转存为 ."+legacyOfficeTarget(ext)+" 后重传")
default:
if isImage(ext) {
doc.Warnings = append(doc.Warnings, "图片不走文本解析(OCR 为 MinerU 深度档,远期)")
block, warn, ok := vision.PrepareImage(filename, mime, content)
if warn != "" {
doc.Warnings = append(doc.Warnings, warn)
}
if ok {
doc.Meta.Parser = "builtin:image"
doc.Blocks = []core.DocBlock{block}
doc.Meta.Confidence = 1
}
} else {
// 未知类型按纯文本尝试(二进制内容会产生乱码 warning)。
text := string(content)
@@ -440,6 +449,7 @@ func ParsersDescription() []core.ParserDesc {
{ID: "builtin:xlsx", Formats: []string{"xlsx", "xlsm"}, Source: core.SourceBuiltin},
{ID: "builtin:pptx", Formats: []string{"pptx", "pptm", "ppsx"}, Source: core.SourceBuiltin},
{ID: "builtin:legacy-office", Formats: []string{"doc", "xls", "ppt"}, Source: core.SourceBuiltin, Note: "不解析;提示转存现代格式后重传"},
{ID: "builtin:image", Formats: []string{"png", "jpg", "jpeg", "gif", "webp"}, Source: core.SourceBuiltin, Note: "随当前轮消息以图像发送;历史轮只留文字占位"},
{ID: "mineru", Formats: []string{"pdf", "png", "jpg"}, Source: core.SourcePlugin, Note: "远期可选:深度解析与交叉验证(默认不部署)"},
}
}
+54 -8
View File
@@ -12,6 +12,9 @@ import (
const OwnerDefault = store.OwnerDefault
const DefaultProfileID = "default"
const FastProfileID = "fast"
const ReasoningProfileID = "reasoning"
const WritingProfileID = "writing"
// defaultSystemPrompt 默认画像不再携带常驻提示词:基座(身份/行为/谨慎/工具/风格)
// 与环境段由提示词库组装(prompts 库 base.*/env.*);文档任务输出要求已技能化
@@ -22,7 +25,7 @@ func DefaultProfile() core.AgentProfile {
return core.AgentProfile{
ID: DefaultProfileID,
Name: "通用助手",
Description: "内置默认画像:全部工具可用,HTTP 工具执行前确认(T0 基线)。",
Description: "内置默认画像:全部工具可用,HTTP 工具执行前确认(T0 基线)。对话模型由系统设置决定。",
Builtin: true,
Source: "builtin",
SystemPrompt: defaultSystemPrompt,
@@ -35,19 +38,57 @@ func DefaultProfile() core.AgentProfile {
}
}
// BuiltinProfiles 是启动时幂等写入的全部内置画像。模型名是网关模型 ID,
// 端点与密钥沿用系统设置。轻量档留空则回落设置里的轻量模型。
func BuiltinProfiles() []core.AgentProfile {
fast := DefaultProfile()
fast.ID = FastProfileID
fast.Name = "快速助手"
fast.Description = "内置画像:对话和轻量任务都用 qwen3.8-flash,适合短问答和改写。"
fast.Model = core.ModelRoute{Chat: "qwen3.8-flash", Lightweight: "qwen3.8-flash"}
reasoning := DefaultProfile()
reasoning.ID = ReasoningProfileID
reasoning.Name = "深度推理"
reasoning.Description = "内置画像:对话用 ZHIPU/GLM-5.3-FlashX。分诊和校验仍走系统轻量模型。"
reasoning.Model = core.ModelRoute{Chat: "ZHIPU/GLM-5.3-FlashX"}
writing := DefaultProfile()
writing.ID = WritingProfileID
writing.Name = "写作与文档"
writing.Description = "内置画像:对话用 ZHIPU/GLM-5.3-Flash。分诊和校验仍走系统轻量模型。"
writing.Model = core.ModelRoute{Chat: "ZHIPU/GLM-5.3-Flash"}
return []core.AgentProfile{DefaultProfile(), fast, reasoning, writing}
}
func builtinID(id string) bool {
for _, p := range BuiltinProfiles() {
if p.ID == id {
return true
}
}
return false
}
type Service struct{ st *store.Store }
func New(st *store.Store) *Service { return &Service{st: st} }
// EnsureDefault 幂等写入内置画像;内置画像只读(Save 拒绝编辑),库中与内置定义
// EnsureDefault 幂等写入全部内置画像;内置画像只读(Save 拒绝编辑),库中与内置定义
// 有差异的行只可能来自更早版本的内置定义,因此差异时升级为当前内置定义。
func (s *Service) EnsureDefault() error {
if cur, err := s.st.GetProfile(OwnerDefault, DefaultProfileID); err == nil {
if reflect.DeepEqual(cur, DefaultProfile()) {
return nil
for _, p := range BuiltinProfiles() {
if cur, err := s.st.GetProfile(OwnerDefault, p.ID); err == nil {
if reflect.DeepEqual(cur, p) {
continue
}
}
if err := s.st.SaveProfile(p); err != nil {
return err
}
}
return s.st.SaveProfile(DefaultProfile())
return nil
}
func (s *Service) List(owner string) ([]core.AgentProfile, error) { return s.st.ListProfiles(owner) }
@@ -62,7 +103,7 @@ func (s *Service) Get(owner, id string) (core.AgentProfile, error) {
}
func (s *Service) Save(owner string, p core.AgentProfile) error {
if p.ID == DefaultProfileID {
if builtinID(p.ID) {
return errReadOnly("profiles: builtin profile is read-only; duplicate and modify")
}
if p.Tools.Allow == nil {
@@ -72,7 +113,12 @@ func (s *Service) Save(owner string, p core.AgentProfile) error {
return s.st.SaveProfile(p)
}
func (s *Service) Delete(owner, id string) error { return s.st.DeleteProfile(owner, id) }
func (s *Service) Delete(owner, id string) error {
if builtinID(id) {
return errReadOnly("profiles: builtin profile is read-only")
}
return s.st.DeleteProfile(owner, id)
}
// SaveAccountChecked is the user-facing write path. Builtin profiles remain
// read-only and the Store binds the write to the actor's effective account.
+52
View File
@@ -0,0 +1,52 @@
package profiles
import (
"path/filepath"
"testing"
"agent-system-backend/internal/store"
core "agent-system-core"
)
func TestBuiltinProfilesRoundTripAndReadOnly(t *testing.T) {
st, err := store.OpenSQLite(filepath.Join(t.TempDir(), "p.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { st.Close() })
svc := New(st)
if err := svc.EnsureDefault(); err != nil {
t.Fatal(err)
}
if err := svc.EnsureDefault(); err != nil {
t.Fatal(err)
}
want := map[string]string{
DefaultProfileID: "",
FastProfileID: "qwen3.8-flash",
ReasoningProfileID: "ZHIPU/GLM-5.3-FlashX",
WritingProfileID: "ZHIPU/GLM-5.3-Flash",
}
for id, model := range want {
got, err := svc.Get(OwnerDefault, id)
if err != nil {
t.Fatal(err)
}
if !got.Builtin || got.Model.Chat != model {
t.Fatalf("%s: builtin=%v model=%q", id, got.Builtin, got.Model.Chat)
}
if err := svc.Save("local", core.AgentProfile{ID: id, Name: "改"}); err == nil {
t.Fatalf("save %s should be rejected", id)
}
if err := svc.Delete("local", id); err == nil {
t.Fatalf("delete %s should be rejected", id)
}
}
listed, err := svc.List(OwnerDefault)
if err != nil {
t.Fatal(err)
}
if len(listed) != len(want) {
t.Fatalf("listed %d", len(listed))
}
}
+4 -1
View File
@@ -17,6 +17,7 @@ import (
"agent-system-backend/internal/ctxasm"
"agent-system-backend/internal/prompts"
"agent-system-backend/internal/quality"
"agent-system-backend/internal/vision"
core "agent-system-core"
)
@@ -228,7 +229,8 @@ func weekdayCN(t time.Time) string {
func (l *LLMRunner) Execute(rc RunCtx) error {
eff := rc.Settings()
prof := rc.Profile()
route := ResolveChatModel(prof, eff)
hasImages := vision.HasVisualInput(rc.Attachments(), currentUserText(rc))
route := ResolveVisionModel(prof, eff, hasImages)
if route.Model == "" || route.Base == "" {
return fmt.Errorf("llm: 主模型未配置(当前解析:model=%q base=%q;检查 settings.models.chat 与画像 ModelRoute;演示模式可开 runtime.mock)", route.Model, route.Base)
}
@@ -275,6 +277,7 @@ func (l *LLMRunner) Execute(rc RunCtx) error {
Surfaces: rc.Surfaces(),
ThinkReplay: thinkReplayEnabled(eff, route.Model),
})
vision.AttachLinkedImages(rc.Context(), msgs)
tail := ctxasm.Tail{
Instructions: eff.GetString("context.userInstructions", ""),
History: hist,
+10 -1
View File
@@ -126,6 +126,15 @@ func (t directTool) Execute(ctx context.Context, args json.RawMessage) (agenttoo
return t.run(ctx, args), nil
}
func messagesHaveImages(msgs []agentllm.Message) bool {
for _, m := range msgs {
if len(m.Images) > 0 {
return true
}
}
return false
}
func (l *LLMRunner) runDirect(rc RunCtx, stream agentllm.StreamFunc, model string, msgs []agentllm.Message, decls []agentllm.ToolDecl, streamOpts agentllm.StreamOptions, cardflow bool, eff core.EffectiveSettings, maxTurns, maxTools int, refresh turnRefresh) error {
st := &directRun{l: l, rc: rc, eff: eff, cardflow: cardflow, started: l.now(), inflight: map[string]*toolSlot{}}
// 记忆端口(N3 WP5):runCtx 暴露 store 时接线(注入由循环按请求完成)。
@@ -161,7 +170,7 @@ func (l *LLMRunner) runDirect(rc RunCtx, stream agentllm.StreamFunc, model strin
break
}
}
stream = agentllm.WithFallback(stream, ChatFallbacks(eff), st.noteModelSwitch)
stream = agentllm.WithFallback(stream, ChatFallbacksFor(st.eff, messagesHaveImages(msgs)), st.noteModelSwitch)
defer func() {
if sink, ok := rc.(interface{ AddUsage(core.Usage) }); ok {
st.switchMu.Lock()
+63 -5
View File
@@ -3,6 +3,7 @@ package runtime
import (
"strings"
"agent-system-backend/internal/vision"
core "agent-system-core"
)
@@ -61,12 +62,48 @@ func ResolveChatModel(prof core.AgentProfile, eff core.EffectiveSettings) Resolv
Key: overlayKey(prof.Owner, "models.chat", eff.GetString("models.chat.apiKey", ""))}
}
// ChatFallbacks 解析 models.chat.fallbacks。默认空,不改变现有模型选择。
func ChatFallbacks(eff core.EffectiveSettings) []string {
raw := strings.TrimSpace(eff.GetString("models.chat.fallbacks", ""))
if raw == "" {
return nil
// ResolveVisionModel 在当前轮含图片时核对视觉清单。清单内的模型保持原选择;
// 不在清单内则这一轮改用 models.vision.model,端点和密钥仍沿用主模型。
// 没有图片时与 ResolveChatModel 相同。
func ResolveVisionModel(prof core.AgentProfile, eff core.EffectiveSettings, hasImages bool) ResolvedModel {
chat := ResolveChatModel(prof, eff)
if !hasImages || modelCapable(chat.Model, VisionCapable(eff)) {
return chat
}
name := strings.TrimSpace(eff.GetString("models.vision.model", ""))
if name == "" {
name = vision.DefaultVisionModel
}
if name == chat.Model {
return chat
}
chat.Model = name
return chat
}
// VisionCapable 解析 models.vision.capable。空字符串用内置默认清单。
func VisionCapable(eff core.EffectiveSettings) []string {
raw := strings.TrimSpace(eff.GetString("models.vision.capable", ""))
if raw == "" {
raw = vision.DefaultVisionCapable
}
return splitModelList(raw)
}
func modelCapable(name string, capable []string) bool {
name = strings.TrimSpace(name)
if name == "" {
return false
}
for _, item := range capable {
if item == name {
return true
}
}
return false
}
func splitModelList(raw string) []string {
var out []string
seen := map[string]bool{}
for _, part := range strings.Split(raw, ",") {
@@ -80,6 +117,27 @@ func ChatFallbacks(eff core.EffectiveSettings) []string {
return out
}
// ChatFallbacks 解析 models.chat.fallbacks。默认空,不改变现有模型选择。
func ChatFallbacks(eff core.EffectiveSettings) []string {
return splitModelList(eff.GetString("models.chat.fallbacks", ""))
}
// ChatFallbacksFor 在图片轮只保留视觉清单内的回退模型,避免退到不能看图的模型。
func ChatFallbacksFor(eff core.EffectiveSettings, hasImages bool) []string {
all := ChatFallbacks(eff)
if !hasImages {
return all
}
capable := VisionCapable(eff)
var out []string
for _, name := range all {
if modelCapable(name, capable) {
out = append(out, name)
}
}
return out
}
// ResolveLightweightModel 解析轻量模型:画像 route > 设置层轻量档 > 回落主档。
func ResolveLightweightModel(prof core.AgentProfile, eff core.EffectiveSettings) ResolvedModel {
m := eff.GetString("models.lightweight.model", "")
+32
View File
@@ -58,3 +58,35 @@ func TestChatFallbacksDefaultEmpty(t *testing.T) {
t.Fatalf("%#v", got)
}
}
func TestResolveVisionModel(t *testing.T) {
eff := effOf(map[string]any{"models": map[string]any{"chat": map[string]any{
"model": "deepseek-v4.1-flash", "baseUrl": "https://gw.example/v1", "apiKey": "k",
"fallbacks": "plain-text,ZHIPU/GLM-5.3-FlashX",
}, "vision": map[string]any{
"model": "qwen3.8-flash",
"capable": "deepseek-v4.1-flash,qwen3.8-flash,ZHIPU/GLM-5.3-FlashX",
}}})
prof := core.AgentProfile{}
got := ResolveVisionModel(prof, eff, false)
if got.Model != "deepseek-v4.1-flash" {
t.Fatalf("no image should keep chat model: %+v", got)
}
got = ResolveVisionModel(prof, eff, true)
if got.Model != "deepseek-v4.1-flash" || got.Base != "https://gw.example/v1" || got.Key != "k" {
t.Fatalf("capable model should stay: %+v", got)
}
prof.Model.Chat = "plain-text"
got = ResolveVisionModel(prof, eff, true)
if got.Model != "qwen3.8-flash" || got.Base != "https://gw.example/v1" || got.Key != "k" {
t.Fatalf("incapable model should swap, keeping endpoint: %+v", got)
}
fb := ChatFallbacksFor(eff, true)
if len(fb) != 1 || fb[0] != "ZHIPU/GLM-5.3-FlashX" {
t.Fatalf("image fallbacks %#v", fb)
}
fb = ChatFallbacksFor(eff, false)
if len(fb) != 2 || fb[0] != "plain-text" {
t.Fatalf("text fallbacks %#v", fb)
}
}
+8 -2
View File
@@ -39,8 +39,12 @@ func SystemDefaults() map[string]any {
"fallbacks": "", // 逗号分隔;空 = 不回退
},
"lightweight": map[string]any{"baseUrl": "", "apiKey": "", "model": ""},
"embedding": map[string]any{"baseUrl": "", "apiKey": "", "model": ""},
"rerank": map[string]any{"baseUrl": "", "apiKey": "", "model": ""},
"vision": map[string]any{
"model": "qwen3.8-flash",
"capable": "deepseek-v4.1-flash,qwen3.8-flash,ZHIPU/GLM-5.3-FlashX,ZHIPU/GLM-5.3-Flash",
},
"embedding": map[string]any{"baseUrl": "", "apiKey": "", "model": ""},
"rerank": map[string]any{"baseUrl": "", "apiKey": "", "model": ""},
},
"tools": map[string]any{
"tavily": map[string]any{"apiKeys": "", "endpoint": ""}, // 网络搜索(逗号分隔密钥池;空 = 不注册搜索工具)
@@ -100,6 +104,8 @@ func Schema() []core.SettingDesc {
{Key: "models.lightweight.baseUrl", Group: "models", Label: "轻量模型 Base URL", Type: core.SettingString, Default: "", Effect: core.EffectHot, Scope: core.SettingScopeSystem, Description: "标题生成/澄清等轻任务;空则回落主模型"},
{Key: "models.lightweight.apiKey", Group: "models", Label: "轻量模型 API Key", Type: core.SettingSecret, Default: "", Effect: core.EffectHot, Scope: core.SettingScopeSystem},
{Key: "models.lightweight.model", Group: "models", Label: "轻量模型名称", Type: core.SettingString, Default: "", Effect: core.EffectNewRun, Scope: core.SettingScopeSystem},
{Key: "models.vision.model", Group: "models", Label: "视觉兜底模型", Type: core.SettingString, Default: "qwen3.8-flash", Effect: core.EffectNewRun, Scope: core.SettingScopeSystem, Description: "当前轮含图片且主模型不在视觉清单内时改用此模型。端点与密钥沿用主模型"},
{Key: "models.vision.capable", Group: "models", Label: "可看图模型", Type: core.SettingString, Default: "deepseek-v4.1-flash,qwen3.8-flash,ZHIPU/GLM-5.3-FlashX,ZHIPU/GLM-5.3-Flash", Effect: core.EffectNewRun, Scope: core.SettingScopeSystem, Description: "逗号分隔。当前轮有图片时,主模型和回退链只保留清单内的名字。空则使用内置默认清单"},
{Key: "models.lightweight.maxOutputTokens", Group: "models", Label: "轻量模型输出上限", Type: core.SettingNumber, Default: 0, Effect: core.EffectNewRun, Scope: core.SettingScopeSystem, Description: "单次回复的 max_tokens。0 = 不限制,由模型服务端决定"},
{Key: "models.embedding.baseUrl", Group: "models", Label: "Embedding Base URL", Type: core.SettingString, Default: "", Effect: core.EffectHot, Scope: core.SettingScopeSystem, Description: "RAG/记忆检索用(向量管线远期启用)"},
{Key: "models.embedding.apiKey", Group: "models", Label: "Embedding API Key", Type: core.SettingSecret, Default: "", Effect: core.EffectHot, Scope: core.SettingScopeSystem},
+193
View File
@@ -0,0 +1,193 @@
package vision
import (
"context"
"encoding/base64"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"strings"
"time"
agentllm "agent-native/runtime/llm"
)
var (
errPrivateAddress = errors.New("地址不在允许范围")
errNotImage = errors.New("响应不是受支持的图片")
errTooLarge = errors.New("超过单图大小上限")
)
// FetchImage 下载一张公网图片。拒绝回环、私网、链路本地和云元数据地址,
// 并把连接钉在解析到的公网 IP 上,避免 DNS 重绑定。不走环境变量里的代理。
func FetchImage(ctx context.Context, rawURL string) (agentllm.Image, error) {
if err := validateImageURL(ctx, rawURL); err != nil {
return agentllm.Image{}, err
}
client := &http.Client{
Timeout: 20 * time.Second,
CheckRedirect: func(req *http.Request, via []*http.Request) error {
if len(via) >= 3 {
return errors.New("重定向过多")
}
if err := validateImageURL(req.Context(), req.URL.String()); err != nil {
return err
}
return nil
},
Transport: &http.Transport{
Proxy: nil,
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
host, port, err := net.SplitHostPort(addr)
if err != nil {
return nil, err
}
ip, err := pickPublicIP(ctx, host)
if err != nil {
return nil, err
}
var d net.Dialer
d.Timeout = 10 * time.Second
return d.DialContext(ctx, network, net.JoinHostPort(ip.String(), port))
},
ResponseHeaderTimeout: 10 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
},
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
if err != nil {
return agentllm.Image{}, err
}
req.Header.Set("User-Agent", "agent-system-vision/1.0")
resp, err := client.Do(req)
if err != nil {
if errors.Is(err, errPrivateAddress) {
return agentllm.Image{}, errPrivateAddress
}
return agentllm.Image{}, fmt.Errorf("下载失败")
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return agentllm.Image{}, fmt.Errorf("下载失败")
}
body, err := io.ReadAll(io.LimitReader(resp.Body, MaxImageBytes+1))
if err != nil {
return agentllm.Image{}, fmt.Errorf("下载失败")
}
if len(body) > MaxImageBytes {
return agentllm.Image{}, errTooLarge
}
kind := MIMEOf(body)
if kind == "" || !contentTypeOK(resp.Header.Get("Content-Type"), kind) {
return agentllm.Image{}, errNotImage
}
// 只回传字节。URL 留给网关自己去拉会绕过这里的地址校验。
return agentllm.Image{
MIME: kind,
Data: base64.StdEncoding.EncodeToString(body),
}, nil
}
func publicReason(err error) string {
switch {
case err == nil:
return ""
case errors.Is(err, errPrivateAddress):
return errPrivateAddress.Error()
case errors.Is(err, errNotImage):
return errNotImage.Error()
case errors.Is(err, errTooLarge):
return errTooLarge.Error()
default:
return "下载失败"
}
}
func validateImageURL(ctx context.Context, raw string) error {
u, err := url.Parse(raw)
if err != nil || u.Host == "" {
return errPrivateAddress
}
if u.User != nil {
return errPrivateAddress
}
switch strings.ToLower(u.Scheme) {
case "http", "https":
default:
return errPrivateAddress
}
if _, err := pickPublicIP(ctx, u.Hostname()); err != nil {
return errPrivateAddress
}
return nil
}
func pickPublicIP(ctx context.Context, host string) (net.IP, error) {
if ip := net.ParseIP(host); ip != nil {
if err := checkIP(ip); err != nil {
return nil, err
}
return ip, nil
}
if host == "" || strings.EqualFold(host, "localhost") {
return nil, errPrivateAddress
}
addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
if err != nil || len(addrs) == 0 {
return nil, errPrivateAddress
}
var first net.IP
for _, addr := range addrs {
if err := checkIP(addr.IP); err != nil {
return nil, err
}
if first == nil {
first = addr.IP
}
}
if first == nil {
return nil, errPrivateAddress
}
return first, nil
}
var (
cgnat = mustCIDR("100.64.0.0/10")
)
func mustCIDR(s string) *net.IPNet {
_, n, err := net.ParseCIDR(s)
if err != nil {
panic(err)
}
return n
}
func checkIP(ip net.IP) error {
if ip == nil {
return errPrivateAddress
}
if v4 := ip.To4(); v4 != nil {
ip = v4
}
switch {
case ip.IsLoopback(), ip.IsPrivate(), ip.IsLinkLocalUnicast(), ip.IsLinkLocalMulticast(),
ip.IsMulticast(), ip.IsUnspecified(), cgnat.Contains(ip):
return errPrivateAddress
default:
return nil
}
}
func contentTypeOK(header, sniffed string) bool {
ct := strings.ToLower(strings.TrimSpace(strings.Split(header, ";")[0]))
switch ct {
case "", "application/octet-stream", "binary/octet-stream":
return sniffed != ""
default:
return ct == sniffed
}
}
+136
View File
@@ -0,0 +1,136 @@
package vision
import (
"encoding/base64"
"fmt"
"path/filepath"
"strings"
agentllm "agent-native/runtime/llm"
core "agent-system-core"
)
// PrepareImage 把上传的图片收成一条 image-ref。Ref 是标准 base64(无换行)。
// 不支持的类型和超限图片返回警告且 ok 为 false,调用方必须把警告展示出来。
func PrepareImage(filename, mime string, content []byte) (core.DocBlock, string, bool) {
ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(filename), "."))
kind, known := sendableExt[ext]
if !known {
return core.DocBlock{}, fmt.Sprintf("图片 .%s 不作为图像发送(请转为 png、jpeg、gif 或 webp)", ext), false
}
if len(content) == 0 {
return core.DocBlock{}, "图片内容为空,本轮不发送图像", false
}
if len(content) > MaxImageBytes {
return core.DocBlock{}, fmt.Sprintf("图片超过 %dMB,本轮不发送图像(请压缩后重传)", MaxImageBytes>>20), false
}
detected := sniffImage(content)
if detected == "" || detected != kind {
return core.DocBlock{}, "图片内容与扩展名不符,本轮不发送图像", false
}
if mime != "" && !strings.EqualFold(strings.TrimSpace(strings.Split(mime, ";")[0]), kind) && !strings.EqualFold(mime, "application/octet-stream") {
return core.DocBlock{}, "图片类型与内容不符,本轮不发送图像", false
}
return core.DocBlock{
Type: core.BlockImageRef,
Ref: base64.StdEncoding.EncodeToString(content),
Text: filename,
}, "图片按图像发送给模型,不转成文字", true
}
// ImagesFromDoc 取出文档里已编码的图片。MIME 来自文件名,不信任外部声明。
func ImagesFromDoc(doc core.ParsedDocument) []agentllm.Image {
ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(doc.Meta.Filename), "."))
kind := sendableExt[ext]
if kind == "" {
return nil
}
var out []agentllm.Image
for _, b := range doc.Blocks {
if b.Type != core.BlockImageRef || strings.TrimSpace(b.Ref) == "" {
continue
}
out = append(out, agentllm.Image{MIME: kind, Data: b.Ref})
}
return out
}
// HasVisualInput 判断这一轮是否带图:已解析的图片附件,或正文里的图片链接。
func HasVisualInput(docs []core.ParsedDocument, text string) bool {
for _, doc := range docs {
if len(ImagesFromDoc(doc)) > 0 {
return true
}
}
return len(ImageLinks(text)) > 0
}
// Admit 按张数和总字节数收下一批图片。被拒绝的每一项都有一句说明。
func Admit(existing []agentllm.Image, incoming []agentllm.Image) (kept []agentllm.Image, notes []string) {
count, total := usage(existing)
for _, img := range incoming {
n := decodedLen(img.Data)
switch {
case n <= 0:
notes = append(notes, "一张图片没有可用内容,已跳过")
case n > MaxImageBytes:
notes = append(notes, fmt.Sprintf("一张图片超过 %dMB,已跳过", MaxImageBytes>>20))
case count >= MaxImages:
notes = append(notes, fmt.Sprintf("本轮图片超过 %d 张,其余未发送", MaxImages))
return kept, notes
case total+n > MaxTotalBytes:
notes = append(notes, fmt.Sprintf("本轮图片合计超过 %dMB,其余未发送", MaxTotalBytes>>20))
return kept, notes
default:
kept = append(kept, img)
count++
total += n
}
}
return kept, notes
}
func usage(images []agentllm.Image) (count, total int) {
for _, img := range images {
n := decodedLen(img.Data)
if n > 0 {
count++
total += n
}
}
return count, total
}
func decodedLen(data string) int {
raw, err := base64.StdEncoding.DecodeString(data)
if err != nil {
return 0
}
return len(raw)
}
var sendableExt = map[string]string{
"png": "image/png",
"jpg": "image/jpeg",
"jpeg": "image/jpeg",
"gif": "image/gif",
"webp": "image/webp",
}
func sniffImage(b []byte) string {
switch {
case len(b) >= 8 && string(b[:8]) == "\x89PNG\r\n\x1a\n":
return "image/png"
case len(b) >= 3 && b[0] == 0xFF && b[1] == 0xD8 && b[2] == 0xFF:
return "image/jpeg"
case len(b) >= 6 && (string(b[:6]) == "GIF87a" || string(b[:6]) == "GIF89a"):
return "image/gif"
case len(b) >= 12 && string(b[:4]) == "RIFF" && string(b[8:12]) == "WEBP":
return "image/webp"
default:
return ""
}
}
// MIMEOf 返回嗅探结果,供下载后的响应校验。
func MIMEOf(content []byte) string { return sniffImage(content) }
+90
View File
@@ -0,0 +1,90 @@
package vision
import (
"context"
"fmt"
"regexp"
"strings"
agentllm "agent-native/runtime/llm"
)
// 只认路径以图片扩展名结尾的 http(s) 链接。不根据任意 URL 去探测,
// 避免把用户消息里的普通网页都拿去下载。
var imageURLRe = regexp.MustCompile(`(?i)https?://[^\s<>"',。、;:]+\.(?:png|jpe?g|gif|webp)(?:\?[^\s<>"',。、;:]*)?`)
// ImageLinks 按出现顺序返回去重后的图片链接。
func ImageLinks(text string) []string {
seen := map[string]bool{}
var out []string
for _, raw := range imageURLRe.FindAllString(text, -1) {
u := strings.TrimRight(raw, ".,;:)]>\"'")
if u == "" || seen[u] {
continue
}
seen[u] = true
out = append(out, u)
}
return out
}
// AttachLinkedImages 下载最后一条用户消息里的图片链接,并写进该消息的 Images。
// 下载失败、超限或地址被拒绝时,把原因追加到正文,不静默丢掉。
func AttachLinkedImages(ctx context.Context, msgs []agentllm.Message) []string {
idx := -1
for i := len(msgs) - 1; i >= 0; i-- {
if msgs[i].Role == agentllm.RoleUser {
idx = i
break
}
}
if idx < 0 {
return nil
}
links := ImageLinks(msgs[idx].Content)
if len(links) == 0 {
return nil
}
var notes []string
var incoming []agentllm.Image
byData := map[string]string{}
for _, link := range links {
img, err := FetchImage(ctx, link)
if err != nil {
notes = append(notes, fmt.Sprintf("%s 未能下载:%s", link, publicReason(err)))
continue
}
incoming = append(incoming, img)
byData[img.Data] = link
}
kept, dropped := Admit(msgs[idx].Images, incoming)
msgs[idx].Images = append(msgs[idx].Images, kept...)
sent := map[string]bool{}
for _, img := range kept {
if link := byData[img.Data]; link != "" {
notes = append(notes, link+" 已随消息发送")
sent[img.Data] = true
}
}
for _, img := range incoming {
if !sent[img.Data] {
if link := byData[img.Data]; link != "" {
notes = append(notes, link+" 未发送")
}
}
}
notes = append(notes, dropped...)
if len(notes) == 0 {
return nil
}
msgs[idx].Content += "\n\n[图片链接]\n" + strings.Join(prefixLines(notes), "\n") + "\n"
return notes
}
func prefixLines(notes []string) []string {
out := make([]string, len(notes))
for i, n := range notes {
out[i] = "- " + n
}
return out
}
+18
View File
@@ -0,0 +1,18 @@
// Package vision 负责「这一轮要不要把图发给模型」:能力清单、图片字节预算、
// 以及用户粘贴的图片链接的受控下载。模型名和端点仍由 runtime 的路由解析。
package vision
// 默认可看图清单与兜底模型。网关路由变化时改设置 models.vision.capable,
// 不必改代码。空设置回落到这里。
const (
DefaultVisionModel = "qwen3.8-flash"
DefaultVisionCapable = "deepseek-v4.1-flash,qwen3.8-flash,ZHIPU/GLM-5.3-FlashX,ZHIPU/GLM-5.3-Flash"
)
// 单图与单轮上限。按网关实测的小图请求体留出余量:base64 约膨胀 4/3,
// 8MB 原图约 11MB 请求体,单轮合计 20MB 原图。
const (
MaxImageBytes = 8 << 20
MaxImages = 8
MaxTotalBytes = 20 << 20
)
+120
View File
@@ -0,0 +1,120 @@
package vision
import (
"context"
"encoding/base64"
"net"
"net/http"
"net/http/httptest"
"strings"
"testing"
agentllm "agent-native/runtime/llm"
core "agent-system-core"
)
func TestCheckIP(t *testing.T) {
blocked := []string{"127.0.0.1", "10.1.2.3", "192.168.1.1", "172.16.0.1", "169.254.169.254", "100.64.0.1", "::1", "fc00::1", "0.0.0.0"}
for _, s := range blocked {
if err := checkIP(net.ParseIP(s)); err == nil {
t.Errorf("%s should be blocked", s)
}
}
if err := checkIP(net.ParseIP("8.8.8.8")); err != nil {
t.Fatal(err)
}
if err := checkIP(net.ParseIP("1.1.1.1")); err != nil {
t.Fatal(err)
}
}
func TestValidateImageURLRejectsUnsafeTargets(t *testing.T) {
ctx := context.Background()
for _, raw := range []string{
"http://127.0.0.1/a.png",
"http://169.254.169.254/latest/a.png",
"http://10.0.0.8/a.jpg",
"http://user:pass@8.8.8.8/a.png",
"file:///tmp/a.png",
"http://localhost/a.png",
} {
if err := validateImageURL(ctx, raw); err == nil {
t.Errorf("accepted %s", raw)
}
}
if err := validateImageURL(ctx, "https://8.8.8.8/a.png"); err != nil {
t.Fatal(err)
}
}
func TestFetchImageRejectsLoopbackServer(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "image/png")
_, _ = w.Write(onePixelPNG)
}))
defer srv.Close()
_, err := FetchImage(context.Background(), srv.URL+"/pixel.png")
if err == nil || !strings.Contains(err.Error(), "允许范围") {
t.Fatalf("err = %v", err)
}
}
func TestImageLinksAndAttachNote(t *testing.T) {
text := "看这张 http://127.0.0.1/a.png。还有 https://cdn.example.com/b.JPG?x=1 和重复 http://127.0.0.1/a.png"
links := ImageLinks(text)
if len(links) != 2 || links[0] != "http://127.0.0.1/a.png" || !strings.Contains(links[1], "b.JPG") {
t.Fatalf("%#v", links)
}
msgs := []agentllm.Message{{Role: agentllm.RoleUser, Content: "请看 http://127.0.0.1/secret.png"}}
notes := AttachLinkedImages(context.Background(), msgs)
if len(notes) == 0 || len(msgs[0].Images) != 0 {
t.Fatalf("notes=%v images=%d", notes, len(msgs[0].Images))
}
if !strings.Contains(msgs[0].Content, "地址不在允许范围") {
t.Fatalf("content missing reason: %s", msgs[0].Content)
}
}
func TestPrepareImageAndBudget(t *testing.T) {
block, warn, ok := PrepareImage("pixel.png", "image/png", onePixelPNG)
if !ok || !strings.Contains(warn, "按图像发送") || block.Type != core.BlockImageRef || block.Ref == "" {
t.Fatalf("ok=%v warn=%q block=%+v", ok, warn, block)
}
if _, warn, ok := PrepareImage("icon.svg", "image/svg+xml", []byte("<svg/>")); ok || !strings.Contains(warn, "不作为图像发送") {
t.Fatalf("svg ok=%v warn=%q", ok, warn)
}
big := make([]byte, MaxImageBytes+1)
if _, warn, ok := PrepareImage("big.png", "image/png", big); ok || !strings.Contains(warn, "超过") {
t.Fatalf("big ok=%v warn=%q", ok, warn)
}
if _, warn, ok := PrepareImage("bad.png", "image/png", []byte("not a png")); ok || !strings.Contains(warn, "不符") {
t.Fatalf("bad ok=%v warn=%q", ok, warn)
}
img := agentllm.Image{MIME: "image/png", Data: block.Ref}
kept, notes := Admit(nil, []agentllm.Image{img})
if len(kept) != 1 || len(notes) != 0 {
t.Fatalf("kept=%d notes=%v", len(kept), notes)
}
many := make([]agentllm.Image, MaxImages+1)
for i := range many {
many[i] = img
}
kept, notes = Admit(nil, many)
if len(kept) != MaxImages || len(notes) == 0 {
t.Fatalf("kept=%d notes=%v", len(kept), notes)
}
raw, err := base64.StdEncoding.DecodeString(block.Ref)
if err != nil || len(raw) != len(onePixelPNG) {
t.Fatal(err)
}
}
// 1x1 PNG。
var onePixelPNG = []byte{
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52,
0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x02, 0x00, 0x00, 0x00, 0x90, 0x77, 0x53,
0xde, 0x00, 0x00, 0x00, 0x0c, 0x49, 0x44, 0x41, 0x54, 0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0x50,
0x0f, 0x00, 0x04, 0x85, 0x01, 0x80, 0x84, 0xa9, 0x8c, 0x21, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45,
0x4e, 0x44, 0xae, 0x42, 0x60, 0x82,
}