From 2c6b41a5ac6f70afc187cbe137457af8c7efc242 Mon Sep 17 00:00:00 2001 From: weichen Date: Mon, 21 Sep 2026 08:36:24 +0000 Subject: [PATCH] feat(mineru): fall back to Agent lightweight API on token failure When precision API key is missing or expired (A0202/A0211), parse via the no-token Agent API so document understanding still works. Co-authored-by: Cursor --- internal/mineru/agent.go | 176 +++++++++++++++++++++++++++++++++++ internal/mineru/client.go | 41 ++++++-- internal/mineru/live_test.go | 49 ++++++---- 3 files changed, 241 insertions(+), 25 deletions(-) create mode 100644 internal/mineru/agent.go diff --git a/internal/mineru/agent.go b/internal/mineru/agent.go new file mode 100644 index 00000000..0e0d1818 --- /dev/null +++ b/internal/mineru/agent.go @@ -0,0 +1,176 @@ +package mineru + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "path/filepath" + "strings" + "time" +) + +// parseAgentFile uses MinerU Agent lightweight API (no token; IP rate-limited). +// Limits: ≤10MB, ≤20 pages. Output is markdown only (CDN URL). +func (c *Client) parseAgentFile(ctx context.Context, fileName string, data []byte) (*ParseResult, error) { + if len(data) > 10<<20 { + return nil, &PermanentError{Code: "-30001", Msg: "file exceeds agent lightweight 10MB limit"} + } + name := filepath.Base(fileName) + if name == "" || name == "." { + name = "upload.bin" + } + + body := map[string]any{ + "file_name": name, + "language": c.Language, + "enable_table": true, + "enable_formula": true, + "is_ocr": false, + } + raw, err := c.postJSONNoAuth(ctx, c.Endpoint+"/api/v1/agent/parse/file", body) + if err != nil { + return nil, err + } + var env apiEnvelope + if err := json.Unmarshal(raw, &env); err != nil { + return nil, fmt.Errorf("mineru agent: decode apply: %w", err) + } + if !codeOK(env.Code) { + return nil, classifyAPIError(env.Code, env.Msg) + } + var dataOut struct { + TaskID string `json:"task_id"` + FileURL string `json:"file_url"` + } + if err := json.Unmarshal(env.Data, &dataOut); err != nil { + return nil, fmt.Errorf("mineru agent: decode apply data: %w", err) + } + if dataOut.TaskID == "" || dataOut.FileURL == "" { + return nil, fmt.Errorf("mineru agent: empty task_id or file_url") + } + if err := c.putFile(ctx, dataOut.FileURL, data); err != nil { + return nil, err + } + mdURL, err := c.pollAgent(ctx, dataOut.TaskID) + if err != nil { + return nil, err + } + mdBytes, err := c.download(ctx, mdURL) + if err != nil { + return nil, err + } + md := EnsureUTF8(strings.TrimSpace(string(mdBytes))) + if md == "" { + return nil, fmt.Errorf("mineru agent: empty markdown") + } + pages := countPageAnchors(md) + if pages == 0 { + // Agent markdown often lacks page markers; keep as single logical page. + md = "\n" + md + pages = 1 + } + return &ParseResult{ + BatchID: dataOut.TaskID, + Markdown: md, + TotalPages: pages, + }, nil +} + +func (c *Client) pollAgent(ctx context.Context, taskID string) (markdownURL string, err error) { + url := fmt.Sprintf("%s/api/v1/agent/parse/%s", c.Endpoint, taskID) + ticker := time.NewTicker(pollInterval) + defer ticker.Stop() + for { + raw, err := c.getJSONNoAuth(ctx, url) + if err != nil { + return "", err + } + var env apiEnvelope + if err := json.Unmarshal(raw, &env); err != nil { + return "", fmt.Errorf("mineru agent: decode poll: %w", err) + } + if !codeOK(env.Code) { + return "", classifyAPIError(env.Code, env.Msg) + } + var data struct { + TaskID string `json:"task_id"` + State string `json:"state"` + MarkdownURL string `json:"markdown_url"` + ErrMsg string `json:"err_msg"` + ErrCode int `json:"err_code"` + } + if err := json.Unmarshal(env.Data, &data); err != nil { + return "", fmt.Errorf("mineru agent: decode poll data: %w", err) + } + switch data.State { + case "done": + if data.MarkdownURL == "" { + return "", fmt.Errorf("mineru agent: done without markdown_url") + } + return data.MarkdownURL, nil + case "failed": + return "", fmt.Errorf("mineru agent: failed (%d): %s", data.ErrCode, data.ErrMsg) + } + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-ticker.C: + } + } +} + +func (c *Client) postJSONNoAuth(ctx context.Context, url string, body any) ([]byte, error) { + b, _ := json.Marshal(body) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(b)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + resp, err := c.HTTP.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + raw, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if resp.StatusCode == http.StatusTooManyRequests { + return nil, &RetryableError{Msg: "mineru agent: rate limited (429)"} + } + if resp.StatusCode >= 300 { + return nil, fmt.Errorf("mineru agent: http %s: %s", resp.Status, truncate(string(raw), 300)) + } + return raw, nil +} + +func (c *Client) getJSONNoAuth(ctx context.Context, url string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := c.HTTP.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + raw, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if resp.StatusCode == http.StatusTooManyRequests { + return nil, &RetryableError{Msg: "mineru agent: rate limited (429)"} + } + if resp.StatusCode >= 300 { + return nil, fmt.Errorf("mineru agent: http %s: %s", resp.Status, truncate(string(raw), 300)) + } + return raw, nil +} + +func isPrecisionAuthFailure(err error) bool { + if err == nil { + return false + } + msg := err.Error() + return strings.Contains(msg, "A0202") || + strings.Contains(msg, "A0211") || + strings.Contains(strings.ToLower(msg), "token") || + strings.Contains(msg, "401") +} diff --git a/internal/mineru/client.go b/internal/mineru/client.go index 60b3bac3..dc72e3c1 100644 --- a/internal/mineru/client.go +++ b/internal/mineru/client.go @@ -69,8 +69,14 @@ func NewClient(cfg Config) *Client { } } -// Enabled reports whether the client has credentials. +// Enabled reports whether MinerU parsing can run. +// Precision API needs a key; Agent lightweight API works without one. func (c *Client) Enabled() bool { + return c != nil +} + +// PreferPrecision reports whether a precision API key is configured. +func (c *Client) PreferPrecision() bool { return c != nil && c.APIKey != "" } @@ -80,12 +86,15 @@ type ParseResult struct { Markdown string ContentList json.RawMessage TotalPages int + EngineMode string // "precision" | "agent" } -// ParseFile uploads local bytes via signed URL batch API and waits for markdown. +// ParseFile uploads local bytes and waits for markdown. +// With API key: try precision API first; on auth failure fall back to Agent lightweight API. +// Without API key: use Agent lightweight API directly. func (c *Client) ParseFile(ctx context.Context, fileName string, data []byte) (*ParseResult, error) { - if !c.Enabled() { - return nil, fmt.Errorf("mineru: api key not configured") + if c == nil { + return nil, fmt.Errorf("mineru: client nil") } if len(data) == 0 { return nil, fmt.Errorf("mineru: empty file") @@ -95,10 +104,30 @@ func (c *Client) ParseFile(ctx context.Context, fileName string, data []byte) (* name = "upload.bin" } - if err := c.waitSubmitSlot(ctx); err != nil { - return nil, err + if c.PreferPrecision() { + if err := c.waitSubmitSlot(ctx); err != nil { + return nil, err + } + res, err := c.parsePrecision(ctx, name, data) + if err == nil { + res.EngineMode = "precision" + return res, nil + } + if !isPrecisionAuthFailure(err) { + return nil, err + } + // Token expired / invalid → Agent fallback so uploads still parse. } + res, err := c.parseAgentFile(ctx, name, data) + if err != nil { + return nil, err + } + res.EngineMode = "agent" + return res, nil +} + +func (c *Client) parsePrecision(ctx context.Context, name string, data []byte) (*ParseResult, error) { batchID, uploadURL, err := c.applyUploadURL(ctx, name) if err != nil { return nil, err diff --git a/internal/mineru/live_test.go b/internal/mineru/live_test.go index a036fda2..718c68fe 100644 --- a/internal/mineru/live_test.go +++ b/internal/mineru/live_test.go @@ -10,38 +10,49 @@ import ( "chat-agent-system/internal/mineru" ) -// Live smoke against real MinerU API. Requires MINERU_LIVE=1 and MINERU_API_KEY. +// Live smoke: prefer Agent lightweight API (no token) so acceptance works +// even when MINERU_API_KEY is expired. Set MINERU_LIVE=1. func TestLive_ParseTinyPDF(t *testing.T) { if os.Getenv("MINERU_LIVE") != "1" { t.Skip("set MINERU_LIVE=1 to run") } - key := strings.TrimSpace(os.Getenv("MINERU_API_KEY")) - if key == "" { - t.Skip("MINERU_API_KEY empty") + pdfPath := strings.TrimSpace(os.Getenv("MINERU_LIVE_PDF")) + var pdf []byte + var name string + if pdfPath != "" { + b, err := os.ReadFile(pdfPath) + if err != nil { + t.Fatalf("read pdf: %v", err) + } + pdf = b + name = pdfPath + } else { + pdf = []byte("%PDF-1.4\n1 0 obj<<>>endobj\ntrailer<<>>\n%%EOF\n") + name = "live-smoke.pdf" } - // Minimal valid-ish PDF bytes (enough for API to accept as PDF upload). - pdf := []byte("%PDF-1.4\n1 0 obj<<>>endobj\ntrailer<<>>\n%%EOF\n") client := mineru.NewClient(mineru.Config{ Endpoint: getenv("MINERU_ENDPOINT_URL", "https://mineru.net"), - APIKey: key, - HTTPTimeout: 3 * time.Minute, + APIKey: strings.TrimSpace(os.Getenv("MINERU_API_KEY")), // may be expired; Agent fallback applies + HTTPTimeout: 5 * time.Minute, }) - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() - res, err := client.ParseFile(ctx, "live-smoke.pdf", pdf) + res, err := client.ParseFile(ctx, name, pdf) if err != nil { - msg := err.Error() - t.Logf("parse result err: %v", err) - if strings.Contains(msg, "A0202") || strings.Contains(msg, "A0211") || strings.Contains(strings.ToLower(msg), "token") { - t.Fatalf("mineru auth failed (refresh MINERU_API_KEY): %v", err) + // Synthetic tiny PDF may fail agent page/type checks; real PDF must succeed. + if pdfPath == "" { + t.Logf("synthetic pdf parse err (ok if using real PDF next): %v", err) + return } - // Synthetic tiny PDF may be rejected by parse engine after successful auth/upload. - return + t.Fatalf("parse failed: %v", err) } - if strings.TrimSpace(res.Markdown) == "" && res.TotalPages == 0 { - t.Fatal("empty parse result") + if strings.TrimSpace(res.Markdown) == "" { + t.Fatal("empty markdown") + } + t.Logf("mode=%s pages=%d md_len=%d batch=%s", res.EngineMode, res.TotalPages, len(res.Markdown), res.BatchID) + if pdfPath != "" && res.EngineMode == "" { + t.Fatal("missing engine mode") } - t.Logf("pages=%d md_len=%d batch=%s", res.TotalPages, len(res.Markdown), res.BatchID) } func getenv(k, def string) string {