// 链接预览端点(提案 2026-09-30-file-preview-and-reserved-capabilities,裁定 D3): // 服务端抓取目标页的 OG/HTML 元数据供前端渲染信息卡。SSRF 硬边界: // 仅 http/https;连接建立前按解析后 IP 拒绝环回/私有/链路本地/多播/未指派地址; // 重定向逐跳同校验;5s 超时;响应体 256KB 截断;Content-Type 必须 text/html。 // 开发/自测可用 AS_URL_PREVIEW_ALLOW_PRIVATE=1 放开私网(默认关,fail-closed)。 package server import ( "context" "fmt" "io" "net" "net/http" "net/url" "os" "regexp" "strings" "sync" "time" ) const ( previewTimeout = 5 * time.Second previewMaxBody = 256 << 10 previewCacheTTL = 15 * time.Minute previewCacheCap = 100 ) type linkPreview struct { URL string `json:"url"` Title string `json:"title,omitempty"` Description string `json:"description,omitempty"` SiteName string `json:"siteName,omitempty"` Favicon string `json:"favicon,omitempty"` Image string `json:"image,omitempty"` FinalURL string `json:"finalUrl,omitempty"` Error string `json:"error,omitempty"` } type previewCacheEntry struct { data linkPreview expiry time.Time } var ( previewCacheMu sync.Mutex previewCache = map[string]previewCacheEntry{} metaOGTitle = regexp.MustCompile(`]+property=["']og:title["'][^>]+content=["']([^"']*)["']`) metaOGName = regexp.MustCompile(`]+name=["']og:title["'][^>]+content=["']([^"']*)["']`) metaOGDesc = regexp.MustCompile(`]+property=["']og:description["'][^>]+content=["']([^"']*)["']`) metaOGSite = regexp.MustCompile(`]+property=["']og:site_name["'][^>]+content=["']([^"']*)["']`) metaOGImg = regexp.MustCompile(`]+property=["']og:image["'][^>]+content=["']([^"']*)["']`) metaDesc = regexp.MustCompile(`]+name=["']description["'][^>]+content=["']([^"']*)["']`) tagTitle = regexp.MustCompile(`]*>([^<]*)`) linkIcon = regexp.MustCompile(`]+rel=["'][^"']*icon[^"']*["'][^>]+href=["']([^"']*)["']`) tagScrip = regexp.MustCompile(`(?is)`) tagStyle = regexp.MustCompile(`(?is)`) ) func (s *Server) urlPreview(w http.ResponseWriter, r *http.Request) { raw := strings.TrimSpace(r.URL.Query().Get("url")) if raw == "" { http.Error(w, "url parameter required", http.StatusBadRequest) return } if cached, ok := previewCacheGet(raw); ok { writeJSON(w, http.StatusOK, cached) return } data := fetchLinkPreview(raw, s.previewAllowPrivate()) previewCachePut(raw, data) writeJSON(w, http.StatusOK, data) } // previewAllowPrivate 环境开关(默认 false):仅供开发/联调,生产 fail-closed。 func (s *Server) previewAllowPrivate() bool { return os.Getenv("AS_URL_PREVIEW_ALLOW_PRIVATE") == "1" } func previewCacheGet(key string) (linkPreview, bool) { previewCacheMu.Lock() defer previewCacheMu.Unlock() e, ok := previewCache[key] if !ok || time.Now().After(e.expiry) { return linkPreview{}, false } return e.data, true } func previewCachePut(key string, data linkPreview) { previewCacheMu.Lock() defer previewCacheMu.Unlock() if len(previewCache) >= previewCacheCap { oldest := "" var oldestT time.Time for k, e := range previewCache { if oldest == "" || e.expiry.Before(oldestT) { oldest, oldestT = k, e.expiry } } if oldest != "" { delete(previewCache, oldest) } } previewCache[key] = previewCacheEntry{data: data, expiry: time.Now().Add(previewCacheTTL)} } // fetchLinkPreview 抓取并解析 OG 元数据;任何校验失败返回带 error 的结构(仍 200, // 前端据此渲染不可用卡),网络错误不冒 5xx。 func fetchLinkPreview(raw string, allowPrivate bool) linkPreview { out := linkPreview{URL: raw} u, err := url.Parse(raw) if err != nil || (u.Scheme != "http" && u.Scheme != "https") { out.Error = "仅支持 http/https 链接" return out } client := &http.Client{Timeout: previewTimeout} if !allowPrivate { dialer := &net.Dialer{Timeout: previewTimeout} client.Transport = &http.Transport{DialContext: guardDial(dialer)} // 重定向逐跳校验:每个 Location 重新走完整校验(含 IP 闸门)。 client.CheckRedirect = func(req *http.Request, via []*http.Request) error { if len(via) >= 5 { return fmt.Errorf("too many redirects") } if req.URL.Scheme != "http" && req.URL.Scheme != "https" { return fmt.Errorf("redirect to non-http scheme rejected") } return nil } } resp, err := client.Get(raw) if err != nil { out.Error = "抓取失败:" + ssrfSafeErr(err) return out } defer resp.Body.Close() if ct := resp.Header.Get("Content-Type"); !strings.Contains(ct, "text/html") { out.Error = "目标不是 HTML 页面(Content-Type: " + ct + ")" return out } body, err := io.ReadAll(io.LimitReader(resp.Body, previewMaxBody)) if err != nil { out.Error = "读取失败:" + ssrfSafeErr(err) return out } out.FinalURL = resp.Request.URL.String() html := tagScrip.ReplaceAllString(string(body), "") html = tagStyle.ReplaceAllString(html, "") if m := metaOGTitle.FindStringSubmatch(html); m != nil { out.Title = decodeHTMLEntities(m[1]) } else if m := metaOGName.FindStringSubmatch(html); m != nil { out.Title = decodeHTMLEntities(m[1]) } else if m := tagTitle.FindStringSubmatch(html); m != nil { out.Title = decodeHTMLEntities(m[1]) } if m := metaOGDesc.FindStringSubmatch(html); m != nil { out.Description = decodeHTMLEntities(m[1]) } else if m := metaDesc.FindStringSubmatch(html); m != nil { out.Description = decodeHTMLEntities(m[1]) } if m := metaOGSite.FindStringSubmatch(html); m != nil { out.SiteName = decodeHTMLEntities(m[1]) } if m := metaOGImg.FindStringSubmatch(html); m != nil { out.Image = resolveRef(out.FinalURL, m[1]) } if m := linkIcon.FindStringSubmatch(html); m != nil { out.Favicon = resolveRef(out.FinalURL, m[1]) } return out } // guardDial 在每个连接建立前校验实际要连的地址(DialContext 收到的是已解析 IP), // 拒绝环回/私有/链路本地/多播/未指派/共享与保留段——校验发生在拨号时点,防 DNS // 重绑定绕过(URL 字符串合法但解析到内网)。 func guardDial(d *net.Dialer) func(ctx context.Context, network, addr string) (net.Conn, error) { return func(ctx context.Context, network, addr string) (net.Conn, error) { if err := validateDialAddr(addr); err != nil { return nil, err } return d.DialContext(ctx, network, addr) } } func validateDialAddr(addr string) error { host, _, err := net.SplitHostPort(addr) if err != nil { return fmt.Errorf("preview dial: bad address %q", addr) } ip := net.ParseIP(host) if ip == nil { return fmt.Errorf("preview dial: non-ip address %q rejected", host) } if ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsMulticast() || ip.IsUnspecified() { return fmt.Errorf("preview dial: forbidden (private/loopback/link-local) address rejected") } if v4 := ip.To4(); v4 != nil { if v4[0] == 240 || v4[0] == 0 { // 保留段 240/4 与 0/8 return fmt.Errorf("preview dial: reserved address rejected") } if v4[0] == 100 && v4[1] >= 64 && v4[1] <= 127 { // CGNAT 共享段 100.64/10 return fmt.Errorf("preview dial: shared address space rejected") } } return nil } func decodeHTMLEntities(s string) string { r := strings.NewReplacer( "&", "&", "<", "<", ">", ">", """, `"`, "'", "'", " ", " ", ) return strings.TrimSpace(r.Replace(s)) } // resolveRef 把 OG 图标/图片的相对引用解析为绝对 URL。 func resolveRef(base, ref string) string { if ref == "" { return "" } b, err := url.Parse(base) if err != nil { return ref } r, err := url.Parse(ref) if err != nil { return ref } return b.ResolveReference(r).String() } // ssrfSafeErr 只保留错误类别,不带内网地址细节回显给客户端。 func ssrfSafeErr(err error) string { msg := err.Error() if strings.Contains(msg, "private") || strings.Contains(msg, "loopback") || strings.Contains(msg, "forbidden") || strings.Contains(msg, "unacceptable") { return "目标地址被安全策略拒绝(内网/保留地址不可抓取)" } if strings.Contains(msg, "timeout") || strings.Contains(msg, "context deadline") { return "目标站点超时" } return "无法访问目标站点" }