Agent-System-Backend/internal/server/urlpreview.go

// 链接预览端点（提案 2026-09-30-file-preview-and-reserved-capabilities，裁定 D3）：
// 服务端抓取目标页的 OG/HTML 元数据供前端渲染信息卡。SSRF 硬边界：
// 仅 http/https；连接建立前按解析后 IP 拒绝环回/私有/链路本地/多播/未指派地址；
// 重定向逐跳同校验；5s 超时；响应体 256KB 截断；Content-Type 必须 text/html。
// 开发/自测可用 AS_URL_PREVIEW_ALLOW_PRIVATE=1 放开私网（默认关，fail-closed）。
package server

import (
	"context"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strings"
	"sync"
	"time"
)

const (
	previewTimeout  = 5 * time.Second
	previewMaxBody  = 256 << 10
	previewCacheTTL = 15 * time.Minute
	previewCacheCap = 100
)

type linkPreview struct {
	URL         string `json:"url"`
	Title       string `json:"title,omitempty"`
	Description string `json:"description,omitempty"`
	SiteName    string `json:"siteName,omitempty"`
	Favicon     string `json:"favicon,omitempty"`
	Image       string `json:"image,omitempty"`
	FinalURL    string `json:"finalUrl,omitempty"`
	Error       string `json:"error,omitempty"`
}

type previewCacheEntry struct {
	data   linkPreview
	expiry time.Time
}

var (
	previewCacheMu sync.Mutex
	previewCache   = map[string]previewCacheEntry{}

	metaOGTitle = regexp.MustCompile(`<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']*)["']`)
	metaOGName  = regexp.MustCompile(`<meta[^>]+name=["']og:title["'][^>]+content=["']([^"']*)["']`)
	metaOGDesc  = regexp.MustCompile(`<meta[^>]+property=["']og:description["'][^>]+content=["']([^"']*)["']`)
	metaOGSite  = regexp.MustCompile(`<meta[^>]+property=["']og:site_name["'][^>]+content=["']([^"']*)["']`)
	metaOGImg   = regexp.MustCompile(`<meta[^>]+property=["']og:image["'][^>]+content=["']([^"']*)["']`)
	metaDesc    = regexp.MustCompile(`<meta[^>]+name=["']description["'][^>]+content=["']([^"']*)["']`)
	tagTitle    = regexp.MustCompile(`<title[^>]*>([^<]*)</title>`)
	linkIcon    = regexp.MustCompile(`<link[^>]+rel=["'][^"']*icon[^"']*["'][^>]+href=["']([^"']*)["']`)
	tagScrip    = regexp.MustCompile(`(?is)<script.*?</script>`)
	tagStyle    = regexp.MustCompile(`(?is)<style.*?</style>`)
)

func (s *Server) urlPreview(w http.ResponseWriter, r *http.Request) {
	raw := strings.TrimSpace(r.URL.Query().Get("url"))
	if raw == "" {
		http.Error(w, "url parameter required", http.StatusBadRequest)
		return
	}
	if cached, ok := previewCacheGet(raw); ok {
		writeJSON(w, http.StatusOK, cached)
		return
	}
	data := fetchLinkPreview(raw, s.previewAllowPrivate())
	previewCachePut(raw, data)
	writeJSON(w, http.StatusOK, data)
}

// previewAllowPrivate 环境开关（默认 false）：仅供开发/联调，生产 fail-closed。
func (s *Server) previewAllowPrivate() bool {
	return os.Getenv("AS_URL_PREVIEW_ALLOW_PRIVATE") == "1"
}

func previewCacheGet(key string) (linkPreview, bool) {
	previewCacheMu.Lock()
	defer previewCacheMu.Unlock()
	e, ok := previewCache[key]
	if !ok || time.Now().After(e.expiry) {
		return linkPreview{}, false
	}
	return e.data, true
}

func previewCachePut(key string, data linkPreview) {
	previewCacheMu.Lock()
	defer previewCacheMu.Unlock()
	if len(previewCache) >= previewCacheCap {
		oldest := ""
		var oldestT time.Time
		for k, e := range previewCache {
			if oldest == "" || e.expiry.Before(oldestT) {
				oldest, oldestT = k, e.expiry
			}
		}
		if oldest != "" {
			delete(previewCache, oldest)
		}
	}
	previewCache[key] = previewCacheEntry{data: data, expiry: time.Now().Add(previewCacheTTL)}
}

// fetchLinkPreview 抓取并解析 OG 元数据；任何校验失败返回带 error 的结构（仍 200，
// 前端据此渲染不可用卡），网络错误不冒 5xx。
func fetchLinkPreview(raw string, allowPrivate bool) linkPreview {
	out := linkPreview{URL: raw}
	u, err := url.Parse(raw)
	if err != nil || (u.Scheme != "http" && u.Scheme != "https") {
		out.Error = "仅支持 http/https 链接"
		return out
	}

	client := &http.Client{Timeout: previewTimeout}
	if !allowPrivate {
		dialer := &net.Dialer{Timeout: previewTimeout}
		client.Transport = &http.Transport{DialContext: guardDial(dialer)}
		// 重定向逐跳校验：每个 Location 重新走完整校验（含 IP 闸门）。
		client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
			if len(via) >= 5 {
				return fmt.Errorf("too many redirects")
			}
			if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
				return fmt.Errorf("redirect to non-http scheme rejected")
			}
			return nil
		}
	}

	resp, err := client.Get(raw)
	if err != nil {
		out.Error = "抓取失败：" + ssrfSafeErr(err)
		return out
	}
	defer resp.Body.Close()
	if ct := resp.Header.Get("Content-Type"); !strings.Contains(ct, "text/html") {
		out.Error = "目标不是 HTML 页面（Content-Type: " + ct + "）"
		return out
	}
	body, err := io.ReadAll(io.LimitReader(resp.Body, previewMaxBody))
	if err != nil {
		out.Error = "读取失败：" + ssrfSafeErr(err)
		return out
	}
	out.FinalURL = resp.Request.URL.String()

	html := tagScrip.ReplaceAllString(string(body), "")
	html = tagStyle.ReplaceAllString(html, "")
	if m := metaOGTitle.FindStringSubmatch(html); m != nil {
		out.Title = decodeHTMLEntities(m[1])
	} else if m := metaOGName.FindStringSubmatch(html); m != nil {
		out.Title = decodeHTMLEntities(m[1])
	} else if m := tagTitle.FindStringSubmatch(html); m != nil {
		out.Title = decodeHTMLEntities(m[1])
	}
	if m := metaOGDesc.FindStringSubmatch(html); m != nil {
		out.Description = decodeHTMLEntities(m[1])
	} else if m := metaDesc.FindStringSubmatch(html); m != nil {
		out.Description = decodeHTMLEntities(m[1])
	}
	if m := metaOGSite.FindStringSubmatch(html); m != nil {
		out.SiteName = decodeHTMLEntities(m[1])
	}
	if m := metaOGImg.FindStringSubmatch(html); m != nil {
		out.Image = resolveRef(out.FinalURL, m[1])
	}
	if m := linkIcon.FindStringSubmatch(html); m != nil {
		out.Favicon = resolveRef(out.FinalURL, m[1])
	}
	return out
}

// guardDial 在每个连接建立前校验实际要连的地址（DialContext 收到的是已解析 IP），
// 拒绝环回/私有/链路本地/多播/未指派/共享与保留段——校验发生在拨号时点，防 DNS
// 重绑定绕过（URL 字符串合法但解析到内网）。
func guardDial(d *net.Dialer) func(ctx context.Context, network, addr string) (net.Conn, error) {
	return func(ctx context.Context, network, addr string) (net.Conn, error) {
		if err := validateDialAddr(addr); err != nil {
			return nil, err
		}
		return d.DialContext(ctx, network, addr)
	}
}

func validateDialAddr(addr string) error {
	host, _, err := net.SplitHostPort(addr)
	if err != nil {
		return fmt.Errorf("preview dial: bad address %q", addr)
	}
	ip := net.ParseIP(host)
	if ip == nil {
		return fmt.Errorf("preview dial: non-ip address %q rejected", host)
	}
	if ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
		ip.IsMulticast() || ip.IsUnspecified() {
		return fmt.Errorf("preview dial: forbidden (private/loopback/link-local) address rejected")
	}
	if v4 := ip.To4(); v4 != nil {
		if v4[0] == 240 || v4[0] == 0 { // 保留段 240/4 与 0/8
			return fmt.Errorf("preview dial: reserved address rejected")
		}
		if v4[0] == 100 && v4[1] >= 64 && v4[1] <= 127 { // CGNAT 共享段 100.64/10
			return fmt.Errorf("preview dial: shared address space rejected")
		}
	}
	return nil
}

func decodeHTMLEntities(s string) string {
	r := strings.NewReplacer(
		"&amp;", "&", "&lt;", "<", "&gt;", ">", "&quot;", `"`, "&#39;", "'", "&nbsp;", " ",
	)
	return strings.TrimSpace(r.Replace(s))
}

// resolveRef 把 OG 图标/图片的相对引用解析为绝对 URL。
func resolveRef(base, ref string) string {
	if ref == "" {
		return ""
	}
	b, err := url.Parse(base)
	if err != nil {
		return ref
	}
	r, err := url.Parse(ref)
	if err != nil {
		return ref
	}
	return b.ResolveReference(r).String()
}

// ssrfSafeErr 只保留错误类别，不带内网地址细节回显给客户端。
func ssrfSafeErr(err error) string {
	msg := err.Error()
	if strings.Contains(msg, "private") || strings.Contains(msg, "loopback") || strings.Contains(msg, "forbidden") || strings.Contains(msg, "unacceptable") {
		return "目标地址被安全策略拒绝（内网/保留地址不可抓取）"
	}
	if strings.Contains(msg, "timeout") || strings.Contains(msg, "context deadline") {
		return "目标站点超时"
	}
	return "无法访问目标站点"
}