From e4c48eb40595d773720736cd66a3785676f8d538 Mon Sep 17 00:00:00 2001
From: weichen <weichen@home.local>
Date: Thu, 1 Oct 2026 05:35:38 +0000
Subject: [PATCH] Wire search into the native backend and add fetch_url with
 SSRF guards

Why: tavily_search/web_search existed as tools but were never
registered on the default execution path (parity-matrix #5 leftover),
and the kernel had no URL fetch capability at all.

Evidence: go vet ./... clean; go test ./... all ok. Fetch cases cover
private/loopback/link-local rejection (127/8, 10/8, 192.168/16,
169.254 metadata endpoint, ::1), file:// scheme rejection, DNS
resolution landing on private space, per-hop redirect validation
rejecting a redirect to loopback, allowed-host policy, HTML stripping
(script/style removed, entities decoded), content-type gating, stub
end-to-end fetch, and absence without FetchConfig (IP normalization
fix included: IPv4-mapped ::ffff:0:0/96 must not swallow all IPv4).
Adapter tests assert Options.Search registers both search tools and
passes queries to the provider, and that neither registers when
unconfigured.

Impact: native.Options gains Search and Fetch; both tool families are
opt-in by the composition root (fail-closed), closing parity-matrix
#5 wiring.

Refs: docs/parity-matrix.md #5; product settings tools.tavily.*
injection point.
---
 adapters/native/backend.go         |  10 ++
 adapters/native/exec_tools_test.go |  61 ++++++++
 runtime/tools/fetch.go             | 225 +++++++++++++++++++++++++++++
 runtime/tools/fetch_test.go        | 154 ++++++++++++++++++++
 4 files changed, 450 insertions(+)
 create mode 100644 runtime/tools/fetch.go
 create mode 100644 runtime/tools/fetch_test.go

diff --git a/adapters/native/backend.go b/adapters/native/backend.go
index 7882c72..5774d64 100644
--- a/adapters/native/backend.go
+++ b/adapters/native/backend.go
@@ -30,9 +30,13 @@ type Options struct {
 	Timeout  time.Duration
 	// Stream 测试注入的流函数；缺省按 APIBase/APIKey 构造 llm.Client。
 	Stream llm.StreamFunc
+	// Search 可选：提供时注册 tavily_search / web_search（密钥由组合根注入）。
+	Search tools.SearchProvider
 	// Exec 可选：提供时注册 run_command / run_python（解释器留空项不注册）。
 	// nil = 不注册执行类工具（fail-closed，与 BashPermission 默认 deny 对齐）。
 	Exec *tools.ExecConfig
+	// Fetch 可选：提供时注册 fetch_url；nil = 不注册（fail-closed）。
+	Fetch *tools.FetchConfig
 	// Log 可选。nil 时丢弃日志。
 	Log *logsys.Logger
 	// ExecutionID 写入日志，不参与执行控制。
@@ -98,6 +102,12 @@ func (b *Backend) Execute(ctx context.Context, req exec.Request) (*exec.Result,
 	var reg *tools.Registry
 	if profile == exec.ToolProfileWorkspace {
 		reg = tools.WorkspaceTools(workDir)
+		if b.opts.Search != nil {
+			tools.RegisterSearch(reg, b.opts.Search)
+		}
+		if b.opts.Fetch != nil {
+			tools.RegisterFetch(reg, *b.opts.Fetch)
+		}
 		if b.opts.Exec != nil {
 			execCfg := *b.opts.Exec
 			for k, v := range req.Env {
diff --git a/adapters/native/exec_tools_test.go b/adapters/native/exec_tools_test.go
index a6a86fd..8a36328 100644
--- a/adapters/native/exec_tools_test.go
+++ b/adapters/native/exec_tools_test.go
@@ -102,3 +102,64 @@ func TestBackendAllowlistCanDisableExecTools(t *testing.T) {
 		}
 	}
 }
+
+// fakeSearchProvider 与 tools 包测试同构的最小搜索替身。
+type fakeSearchProvider struct{ query string }
+
+func (f *fakeSearchProvider) Search(_ context.Context, req tools.SearchRequest) (tools.SearchResponse, error) {
+	f.query = req.Query
+	return tools.SearchResponse{Answer: "ans", Results: []tools.SearchResult{{Title: "T", Content: "C", Score: 0.9}}}, nil
+}
+
+func TestBackendSearchToolsWiredViaOptions(t *testing.T) {
+	provider := &fakeSearchProvider{}
+	stream, c := captureStream(
+		llm.Message{Role: llm.RoleAssistant, StopReason: llm.StopToolCalls, ToolCalls: []llm.ToolCall{{
+			ID: "c1", Name: "web_search", Arguments: `{"query":"golang generics"}`,
+		}}},
+		llm.Message{Role: llm.RoleAssistant, StopReason: llm.StopToolCalls, ToolCalls: []llm.ToolCall{{
+			ID: "c2", Name: "write_file",
+			Arguments: `{"path":"output.md","content":"done"}`,
+		}}},
+		llm.Message{Role: llm.RoleAssistant, Content: "ok", StopReason: llm.StopEnd},
+	)
+	b := New(Options{Model: "fake", Stream: stream, Search: provider})
+	dir := t.TempDir()
+	res, err := b.Execute(context.Background(), exec.Request{
+		WorkDir: dir, Prompt: "search", Targets: []string{"output.md"},
+	})
+	if err != nil || !res.Success {
+		t.Fatalf("execute: err=%v res=%+v logs=%s", err, res, res.Logs)
+	}
+	if provider.query != "golang generics" {
+		t.Fatalf("provider should receive the query, got %q", provider.query)
+	}
+	// 搜索工具确实出现在 decls 里。
+	found := false
+	for _, names := range c.toolDecls {
+		for _, n := range names {
+			if n == "web_search" || n == "tavily_search" {
+				found = true
+			}
+		}
+	}
+	if !found {
+		t.Fatal("search tools must be declared when Options.Search is set")
+	}
+}
+
+func TestBackendSearchToolsAbsentWithoutOptions(t *testing.T) {
+	stream, c := captureStream()
+	b := New(Options{Model: "fake", Stream: stream})
+	_, _ = b.Execute(context.Background(), exec.Request{
+		WorkDir: t.TempDir(), Prompt: "x", Targets: []string{"output.md"},
+		ToolProfile: exec.ToolProfileWorkspace,
+	})
+	for _, names := range c.toolDecls {
+		for _, n := range names {
+			if n == "web_search" || n == "tavily_search" {
+				t.Fatalf("search tools must not register without Options.Search, got %v", names)
+			}
+		}
+	}
+}
diff --git a/runtime/tools/fetch.go b/runtime/tools/fetch.go
new file mode 100644
index 0000000..274bc1c
--- /dev/null
+++ b/runtime/tools/fetch.go
@@ -0,0 +1,225 @@
+package tools
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"html"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+	"time"
+)
+
+// FetchConfig 声明 fetch_url 的运行条件；未注册（组合根未提供）则工具不存在。
+type FetchConfig struct {
+	// Client 可选注入（测试替身）；缺省构造 20s 超时客户端。
+	Client *http.Client
+	// MaxBytes 响应体上限（默认 2 MiB）。
+	MaxBytes int
+	// AllowedHosts 非空时仅允许这些主机（域名或 host:port）；空 = 公网放开，
+	// 但私网/回环/链路本地地址在任何模式下都被拒绝（SSRF 硬边界）。
+	AllowedHosts []string
+	// Resolve 可选注入主机解析（测试替身）；缺省 net.LookupIP。
+	Resolve func(host string) ([]net.IP, error)
+}
+
+const (
+	fetchTimeout  = 20 * time.Second
+	fetchMaxBytes = 2 << 20
+)
+
+func (c *FetchConfig) resolver() func(string) ([]net.IP, error) {
+	if c.Resolve != nil {
+		return c.Resolve
+	}
+	return net.LookupIP
+}
+
+// RegisterFetch 注册 fetch_url 工具（cfg 必须非 nil，由组合根显式提供）。
+func RegisterFetch(r *Registry, cfg FetchConfig) {
+	r.Register(&fetchTool{cfg: cfg})
+}
+
+type fetchTool struct {
+	cfg FetchConfig
+}
+
+func (t *fetchTool) Name() string { return "fetch_url" }
+func (t *fetchTool) Description() string {
+	return "Fetch an http(s) URL and return readable text (HTML stripped, up to 2 MiB, 20s timeout). Private/loopback network targets are always rejected."
+}
+func (t *fetchTool) Parameters() json.RawMessage {
+	return json.RawMessage(`{"type":"object","required":["url"],"properties":{"url":{"type":"string"}}}`)
+}
+
+func (t *fetchTool) Execute(ctx context.Context, args json.RawMessage) (Result, error) {
+	var obj struct {
+		URL string `json:"url"`
+	}
+	if err := json.Unmarshal(args, &obj); err != nil {
+		return Result{}, err
+	}
+	raw := strings.TrimSpace(obj.URL)
+	if raw == "" {
+		return Result{Content: "url is empty", IsError: true}, nil
+	}
+	client := t.cfg.Client
+	if client == nil {
+		client = &http.Client{Timeout: fetchTimeout}
+	}
+	allowed := make(map[string]bool, len(t.cfg.AllowedHosts))
+	for _, h := range t.cfg.AllowedHosts {
+		allowed[strings.ToLower(h)] = true
+	}
+	maxBytes := t.cfg.MaxBytes
+	if maxBytes <= 0 {
+		maxBytes = fetchMaxBytes
+	}
+
+	reqCtx, cancel := context.WithTimeout(ctx, fetchTimeout)
+	defer cancel()
+	req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, raw, nil)
+	if err != nil {
+		return Result{Content: err.Error(), IsError: true}, nil
+	}
+	req.Header.Set("User-Agent", "agent-native-fetch/1.0")
+	var finalURL string
+	resolve := t.cfg.resolver()
+	client2 := client
+	if client2.CheckRedirect == nil {
+		client2 = &http.Client{
+			Transport:     client.Transport,
+			Jar:           client.Jar,
+			Timeout:       client.Timeout,
+			CheckRedirect: func(req *http.Request, via []*http.Request) error {
+				if len(via) >= 5 {
+					return fmt.Errorf("stopped after 5 redirects")
+				}
+				return validateFetchTarget(req.URL, allowed, resolve)
+			},
+		}
+	}
+	if err := validateFetchTarget(req.URL, allowed, resolve); err != nil {
+		return Result{Content: err.Error(), IsError: true}, nil
+	}
+	resp, err := client2.Do(req)
+	if err != nil {
+		return Result{Content: err.Error(), IsError: true}, nil
+	}
+	defer resp.Body.Close()
+	finalURL = resp.Request.URL.String()
+	body, err := io.ReadAll(io.LimitReader(resp.Body, int64(maxBytes)+1))
+	if err != nil {
+		return Result{Content: err.Error(), IsError: true}, nil
+	}
+	if len(body) > maxBytes {
+		body = body[:maxBytes]
+	}
+	ct := resp.Header.Get("Content-Type")
+	text, ok := fetchReadableText(ct, string(body))
+	if !ok {
+		return Result{Content: fmt.Sprintf("unsupported content type %q at %s (only text/html/json/xml/yaml/csv are extracted)", ct, finalURL), IsError: true}, nil
+	}
+	return Result{Content: "url: " + finalURL + "\ncontent-type: " + ct + "\n\n" + text}, nil
+}
+
+// validateFetchTarget 校验 scheme、主机策略与 SSRF 硬边界（私网/回环/链路本地）。
+func validateFetchTarget(u *url.URL, allowed map[string]bool, resolve func(string) ([]net.IP, error)) error {
+	if u.Scheme != "http" && u.Scheme != "https" {
+		return fmt.Errorf("scheme %q not allowed (http/https only)", u.Scheme)
+	}
+	host := u.Hostname()
+	if host == "" {
+		return fmt.Errorf("empty host")
+	}
+	if len(allowed) > 0 && !allowed[strings.ToLower(host)] {
+		return fmt.Errorf("host %q not in allowed hosts", host)
+	}
+	// 解析全部地址：任一命中私网段即拒绝（域名可能同时解析到公私网地址）。
+	addrs, err := resolve(host)
+	if err != nil {
+		return fmt.Errorf("resolve %q: %w", host, err)
+	}
+	for _, ip := range addrs {
+		if ipIsPrivate(ip) {
+			return fmt.Errorf("target %q resolves to private/loopback address %s (SSRF guard)", host, ip)
+		}
+	}
+	return nil
+}
+
+var fetchPrivateNets = mustParseCIDRs(
+	"0.0.0.0/8", "10.0.0.0/8", "127.0.0.0/8", "169.254.0.0/16",
+	"172.16.0.0/12", "192.168.0.0/16", "100.64.0.0/10",
+	"::1/128", "fc00::/7", "fe80::/10",
+)
+
+func mustParseCIDRs(specs ...string) []*net.IPNet {
+	var out []*net.IPNet
+	for _, s := range specs {
+		_, n, err := net.ParseCIDR(s)
+		if err != nil {
+			panic("fetch: bad cidr " + s)
+		}
+		out = append(out, n)
+	}
+	return out
+}
+
+func ipIsPrivate(ip net.IP) bool {
+	// IPv4（含 Go 内部的 ::ffff:a.b.c.d 映射形式）先归一化成 4 字节，
+	// 否则 ::ffff:0:0/96 这类映射网段会误吞所有 IPv4 地址。
+	if v4 := ip.To4(); v4 != nil {
+		ip = v4
+	}
+	if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsUnspecified() {
+		return true
+	}
+	for _, n := range fetchPrivateNets {
+		if n.Contains(ip) {
+			return true
+		}
+	}
+	return false
+}
+
+// fetchReadableText 按内容类型抽取可读文本；HTML 剥脚本样式与标签。
+func fetchReadableText(contentType, body string) (string, bool) {
+	ct := strings.ToLower(strings.TrimSpace(strings.Split(contentType, ";")[0]))
+	if ct == "" {
+		ct = "text/plain" // 无声明按纯文本处理
+	}
+	switch {
+	case ct == "text/html":
+		return htmlToText(body), true
+	case strings.HasPrefix(ct, "text/"),
+		ct == "application/json",
+		ct == "application/xml",
+		strings.HasSuffix(ct, "+json"),
+		strings.HasSuffix(ct, "+xml"),
+		ct == "application/yaml" || ct == "text/yaml",
+		ct == "application/csv" || ct == "text/csv":
+		return body, true
+	default:
+		return "", false
+	}
+}
+
+var (
+	fetchScriptRe = regexp.MustCompile(`(?is)<(script|style|noscript)[^>]*>.*?</(script|style|noscript)>`)
+	fetchTagRe    = regexp.MustCompile(`(?s)<[^>]*>`)
+	fetchSpaceRe  = regexp.MustCompile(`\n{3,}`)
+)
+
+func htmlToText(body string) string {
+	s := fetchScriptRe.ReplaceAllString(body, "")
+	s = fetchTagRe.ReplaceAllString(s, "\n")
+	s = html.UnescapeString(s)
+	s = strings.ReplaceAll(s, "\r", "")
+	s = fetchSpaceRe.ReplaceAllString(s, "\n\n")
+	return strings.TrimSpace(s)
+}
diff --git a/runtime/tools/fetch_test.go b/runtime/tools/fetch_test.go
new file mode 100644
index 0000000..842f8f8
--- /dev/null
+++ b/runtime/tools/fetch_test.go
@@ -0,0 +1,154 @@
+package tools
+
+import (
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+// fakeResolve 固定域名 → 地址（无 DNS 依赖的封闭测试）。
+func fakeResolve(mapping map[string][]net.IP) func(string) ([]net.IP, error) {
+	return func(host string) ([]net.IP, error) {
+		if ips, ok := mapping[host]; ok {
+			return ips, nil
+		}
+		// IP 字面量直接解析。
+		if ip := net.ParseIP(host); ip != nil {
+			return []net.IP{ip}, nil
+		}
+		return nil, &net.DNSError{Err: "no such host", Name: host}
+	}
+}
+
+func fetchReg(cfg FetchConfig) *Registry {
+	reg := NewRegistry("/")
+	RegisterFetch(reg, cfg)
+	return reg
+}
+
+func TestFetchReadableTextUnit(t *testing.T) {
+	out, ok := fetchReadableText("text/html; charset=utf-8", "<html><script>bad()</script><h1>Hi</h1><p>A &amp; B</p></html>")
+	if !ok || !strings.Contains(out, "Hi") || !strings.Contains(out, "A & B") || strings.Contains(out, "bad()") {
+		t.Fatalf("html strip wrong: %q ok=%v", out, ok)
+	}
+	out, ok = fetchReadableText("application/json", `{"x":1}`)
+	if !ok || out != `{"x":1}` {
+		t.Fatalf("json passthrough wrong: %q", out)
+	}
+	_, ok = fetchReadableText("image/png", "binary")
+	if ok {
+		t.Fatal("binary content type must be rejected")
+	}
+	out, ok = fetchReadableText("", "no content type")
+	if !ok || out != "no content type" {
+		t.Fatalf("empty content type should default to text, got %q %v", out, ok)
+	}
+}
+
+func TestFetchURLSSRDGuards(t *testing.T) {
+	reg := fetchReg(FetchConfig{Resolve: fakeResolve(nil)})
+
+	for _, target := range []string{
+		`{"url":"http://127.0.0.1/x"}`,
+		`{"url":"http://10.0.0.5/x"}`,
+		`{"url":"http://192.168.1.1/x"}`,
+		`{"url":"http://169.254.169.254/latest/meta-data"}`,
+		`{"url":"http://[::1]/x"}`,
+		`{"url":"file:///etc/passwd"}`,
+	} {
+		res := runTool(t, reg, "fetch_url", target)
+		if !res.IsError {
+			t.Fatalf("SSRF/scheme guard must reject %s, got %+v", target, res)
+		}
+	}
+
+	// 域名解析到私网同样拒绝。
+	reg2 := fetchReg(FetchConfig{Resolve: fakeResolve(map[string][]net.IP{
+		"internal.example": {net.ParseIP("192.168.0.10")},
+	})})
+	res := runTool(t, reg2, "fetch_url", `{"url":"http://internal.example/x"}`)
+	if !res.IsError || !strings.Contains(res.Content, "SSRF guard") {
+		t.Fatalf("private resolution must be rejected, got %+v", res)
+	}
+}
+
+func TestFetchURLAllowedHostsPolicy(t *testing.T) {
+	reg := fetchReg(FetchConfig{
+		AllowedHosts: []string{"api.example.com"},
+		Resolve:      fakeResolve(map[string][]net.IP{"api.example.com": {net.ParseIP("93.184.216.34")}}),
+	})
+	res := runTool(t, reg, "fetch_url", `{"url":"http://other.example.com/x"}`)
+	if !res.IsError || !strings.Contains(res.Content, "not in allowed hosts") {
+		t.Fatalf("host policy must reject, got %+v", res)
+	}
+	// 允许的主机：校验必须放行（真实网络下可能成功取回，也可能因出口
+	// 受限连接失败——两者都不算策略拦截）。
+	res = runTool(t, reg, "fetch_url", `{"url":"http://api.example.com/x"}`)
+	if strings.Contains(res.Content, "not in allowed") || strings.Contains(res.Content, "SSRF") {
+		t.Fatalf("allowed host should pass validation, got %+v", res)
+	}
+}
+
+func TestFetchURLEndToEndWithStubTransport(t *testing.T) {
+	var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/plain")
+		_, _ = w.Write([]byte("fetched body"))
+	}
+	reg := fetchReg(FetchConfig{
+		Client:  &http.Client{Transport: stubTransport{handler: handler}},
+		Resolve: fakeResolve(map[string][]net.IP{"ok.example": {net.ParseIP("93.184.216.34")}}),
+	})
+	res := runTool(t, reg, "fetch_url", `{"url":"http://ok.example/x"}`)
+	if res.IsError {
+		t.Fatal(res.Content)
+	}
+	if !strings.Contains(res.Content, "fetched body") || !strings.Contains(res.Content, "url: http://ok.example/x") {
+		t.Fatalf("stub fetch wrong: %+v", res)
+	}
+}
+
+type stubTransport struct{ handler http.HandlerFunc }
+
+func (s stubTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	rec := httptest.NewRecorder()
+	s.handler(rec, req)
+	resp := rec.Result()
+	resp.Request = req
+	return resp, nil
+}
+
+func TestFetchURLRedirectToPrivateRejected(t *testing.T) {
+	reg := fetchReg(FetchConfig{
+		Client: &http.Client{Transport: redirectTransport{to: "http://127.0.0.1/evil"}},
+		Resolve: fakeResolve(map[string][]net.IP{
+			"public.example": {net.ParseIP("93.184.216.34")},
+			"127.0.0.1":      {net.ParseIP("127.0.0.1")},
+		}),
+	})
+	res := runTool(t, reg, "fetch_url", `{"url":"http://public.example/redirect"}`)
+	if !res.IsError {
+		t.Fatalf("redirect to private target must be rejected, got %+v", res)
+	}
+}
+
+type redirectTransport struct{ to string }
+
+func (s redirectTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	rec := httptest.NewRecorder()
+	rec.Header().Set("Location", s.to)
+	rec.Code = http.StatusFound
+	resp := rec.Result()
+	resp.Request = req
+	return resp, nil
+}
+
+func TestFetchNotRegisteredByDefault(t *testing.T) {
+	reg := WorkspaceTools(t.TempDir())
+	for _, d := range reg.Decls() {
+		if d.Name == "fetch_url" {
+			t.Fatal("fetch_url must not register without explicit FetchConfig")
+		}
+	}
+}