weichenandCursor
cfde38de31
Why: Uploaded images never reached the model, so a text-only chat model could be asked to read a picture it cannot see. Evidence: go test ./internal/... passed locally before this commit. Image-link rejection is covered by vision package tests. Impact: Current-turn attachments and fetched image links become image input. History keeps a text placeholder. Image turns swap to the configured vision model when the chat model is not in the capable list, and fallbacks are filtered the same way. Refs: Agent-System-Dev feature/four-model-vision Co-authored-by: Cursor <cursoragent@cursor.com>
91 lines
2.3 KiB
Go
91 lines
2.3 KiB
Go
package vision
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"regexp"
|
|
"strings"
|
|
|
|
agentllm "agent-native/runtime/llm"
|
|
)
|
|
|
|
// 只认路径以图片扩展名结尾的 http(s) 链接。不根据任意 URL 去探测，
|
|
// 避免把用户消息里的普通网页都拿去下载。
|
|
var imageURLRe = regexp.MustCompile(`(?i)https?://[^\s<>"'，。、；：]+\.(?:png|jpe?g|gif|webp)(?:\?[^\s<>"'，。、；：]*)?`)
|
|
|
|
// ImageLinks 按出现顺序返回去重后的图片链接。
|
|
func ImageLinks(text string) []string {
|
|
seen := map[string]bool{}
|
|
var out []string
|
|
for _, raw := range imageURLRe.FindAllString(text, -1) {
|
|
u := strings.TrimRight(raw, ".,;:)]>\"'")
|
|
if u == "" || seen[u] {
|
|
continue
|
|
}
|
|
seen[u] = true
|
|
out = append(out, u)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// AttachLinkedImages 下载最后一条用户消息里的图片链接，并写进该消息的 Images。
|
|
// 下载失败、超限或地址被拒绝时，把原因追加到正文，不静默丢掉。
|
|
func AttachLinkedImages(ctx context.Context, msgs []agentllm.Message) []string {
|
|
idx := -1
|
|
for i := len(msgs) - 1; i >= 0; i-- {
|
|
if msgs[i].Role == agentllm.RoleUser {
|
|
idx = i
|
|
break
|
|
}
|
|
}
|
|
if idx < 0 {
|
|
return nil
|
|
}
|
|
links := ImageLinks(msgs[idx].Content)
|
|
if len(links) == 0 {
|
|
return nil
|
|
}
|
|
var notes []string
|
|
var incoming []agentllm.Image
|
|
byData := map[string]string{}
|
|
for _, link := range links {
|
|
img, err := FetchImage(ctx, link)
|
|
if err != nil {
|
|
notes = append(notes, fmt.Sprintf("%s 未能下载：%s", link, publicReason(err)))
|
|
continue
|
|
}
|
|
incoming = append(incoming, img)
|
|
byData[img.Data] = link
|
|
}
|
|
kept, dropped := Admit(msgs[idx].Images, incoming)
|
|
msgs[idx].Images = append(msgs[idx].Images, kept...)
|
|
sent := map[string]bool{}
|
|
for _, img := range kept {
|
|
if link := byData[img.Data]; link != "" {
|
|
notes = append(notes, link+" 已随消息发送")
|
|
sent[img.Data] = true
|
|
}
|
|
}
|
|
for _, img := range incoming {
|
|
if !sent[img.Data] {
|
|
if link := byData[img.Data]; link != "" {
|
|
notes = append(notes, link+" 未发送")
|
|
}
|
|
}
|
|
}
|
|
notes = append(notes, dropped...)
|
|
if len(notes) == 0 {
|
|
return nil
|
|
}
|
|
msgs[idx].Content += "\n\n[图片链接]\n" + strings.Join(prefixLines(notes), "\n") + "\n"
|
|
return notes
|
|
}
|
|
|
|
func prefixLines(notes []string) []string {
|
|
out := make([]string, len(notes))
|
|
for i, n := range notes {
|
|
out[i] = "- " + n
|
|
}
|
|
return out
|
|
}
|