weichenandCursor
1e482023b6
Cache MinerU parse results by blob hash and store project-scoped chunk embeddings for large uploaded materials. Co-authored-by: Cursor <cursoragent@cursor.com>
58 lines
2.3 KiB
SQL
58 lines
2.3 KiB
SQL
-- +goose Up
|
|
-- 材料解析缓存（按 blob 去重）+ 大文档分块向量索引（项目隔离）。
|
|
|
|
CREATE TABLE IF NOT EXISTS asset_parses (
|
|
id BIGSERIAL PRIMARY KEY,
|
|
blob_sha256 TEXT NOT NULL,
|
|
file_name TEXT NOT NULL DEFAULT '',
|
|
engine TEXT NOT NULL DEFAULT 'mineru',
|
|
engine_version TEXT NOT NULL DEFAULT '',
|
|
state TEXT NOT NULL DEFAULT 'pending',
|
|
task_id TEXT NOT NULL DEFAULT '',
|
|
markdown TEXT NOT NULL DEFAULT '',
|
|
content_list JSONB NOT NULL DEFAULT '[]'::jsonb,
|
|
total_pages INT NOT NULL DEFAULT 0,
|
|
error_message TEXT NOT NULL DEFAULT '',
|
|
attempts INT NOT NULL DEFAULT 0,
|
|
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
CONSTRAINT asset_parses_state_chk CHECK (state IN ('pending', 'running', 'done', 'failed'))
|
|
);
|
|
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_asset_parses_blob_engine
|
|
ON asset_parses (blob_sha256, engine);
|
|
CREATE INDEX IF NOT EXISTS idx_asset_parses_state
|
|
ON asset_parses (state, updated_at DESC);
|
|
|
|
CREATE TABLE IF NOT EXISTS material_chunks (
|
|
id BIGSERIAL PRIMARY KEY,
|
|
project_id BIGINT NOT NULL DEFAULT 0,
|
|
blob_sha256 TEXT NOT NULL,
|
|
file_name TEXT NOT NULL DEFAULT '',
|
|
chunk_index INT NOT NULL DEFAULT 0,
|
|
title TEXT NOT NULL DEFAULT '',
|
|
content TEXT NOT NULL,
|
|
page_start INT NOT NULL DEFAULT 0,
|
|
page_end INT NOT NULL DEFAULT 0,
|
|
chunk_hash TEXT NOT NULL,
|
|
embedding vector(1024),
|
|
metadata JSONB NOT NULL DEFAULT '{}'::jsonb,
|
|
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
);
|
|
|
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_material_chunks_hash
|
|
ON material_chunks (chunk_hash);
|
|
CREATE INDEX IF NOT EXISTS idx_material_chunks_project_blob
|
|
ON material_chunks (project_id, blob_sha256);
|
|
CREATE INDEX IF NOT EXISTS idx_material_chunks_vec
|
|
ON material_chunks USING hnsw (embedding vector_cosine_ops);
|
|
|
|
-- +goose Down
|
|
DROP INDEX IF EXISTS idx_material_chunks_vec;
|
|
DROP INDEX IF EXISTS idx_material_chunks_project_blob;
|
|
DROP INDEX IF EXISTS uq_material_chunks_hash;
|
|
DROP TABLE IF EXISTS material_chunks;
|
|
DROP INDEX IF EXISTS idx_asset_parses_state;
|
|
DROP INDEX IF EXISTS uq_asset_parses_blob_engine;
|
|
DROP TABLE IF EXISTS asset_parses;
|