chore(deploy): 三环境 Compose 构建真实 app 镜像并补齐法规库运维
将 dev/test/main 的 app 服务从占位容器改为 Dockerfile 多阶段构建,增加 healthcheck 与 .regulations 只读挂载;新增法规库备份/恢复脚本并同步 env-service 目录初始化与 gitignore。 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -0,0 +1,16 @@
|
||||
.git
|
||||
.cursor
|
||||
.doc
|
||||
docs
|
||||
data
|
||||
web/node_modules
|
||||
web/dist
|
||||
web/.vite
|
||||
bin
|
||||
*.md
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
configs/.env.*
|
||||
agent-transcripts
|
||||
.refs
|
||||
@@ -54,3 +54,7 @@ node_modules/
|
||||
*.o
|
||||
*.a
|
||||
/server
|
||||
|
||||
# Regulations Knowledge Base Data
|
||||
.regulations/
|
||||
regulations/
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
# =============================================================================
|
||||
# Go API Server (agent-dev / agent-test / agent-main)
|
||||
# =============================================================================
|
||||
FROM golang:1.23-bookworm AS builder
|
||||
|
||||
ENV GOTOOLCHAIN=auto
|
||||
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /server ./cmd/server
|
||||
|
||||
FROM debian:bookworm-slim
|
||||
|
||||
RUN set -eux; \
|
||||
apt-get update; \
|
||||
apt-get install -y --no-install-recommends ca-certificates curl; \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
COPY --from=builder /server /app/server
|
||||
|
||||
ENV DATA_DIR=/app/data \
|
||||
HTTP_HOST=0.0.0.0 \
|
||||
HTTP_PORT=8080
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=10s --timeout=5s --retries=6 \
|
||||
CMD curl -fsS "http://127.0.0.1:${HTTP_PORT:-8080}/api/v1/health" || exit 1
|
||||
|
||||
ENTRYPOINT ["/app/server"]
|
||||
CMD ["-env", "/dev/null"]
|
||||
+13
-2
@@ -80,15 +80,19 @@ services:
|
||||
- dev-net
|
||||
|
||||
app:
|
||||
image: alpine:latest
|
||||
build:
|
||||
context: .
|
||||
dockerfile: deploy/app/Dockerfile
|
||||
container_name: shuimu-agent-app-dev
|
||||
command: ["sh", "-c", "echo 'Dev Agent App service is running on :8080' && sleep infinity"]
|
||||
depends_on:
|
||||
system-db:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
ENV: dev
|
||||
HTTP_ADDR: ":${HTTP_PORT:-8080}"
|
||||
HTTP_PORT: "${HTTP_PORT:-8080}"
|
||||
HTTP_HOST: "0.0.0.0"
|
||||
AGENT_BACKEND: ${AGENT_BACKEND:-pi-agent}
|
||||
DATA_DIR: /app/data
|
||||
# 独占业务与对话库
|
||||
APP_DB_DSN: postgres://${SYSTEM_DB_USER:-app}:${SYSTEM_DB_PASSWORD:-dev_secret_db_pass}@system-db:5432/${SYSTEM_DB_NAME:-agent_system_dev}?sslmode=disable
|
||||
@@ -99,10 +103,17 @@ services:
|
||||
AUX_FILE_DSN: file:/var/lib/feishu/outbox_dev.db?_journal_mode=WAL&_busy_timeout=5000
|
||||
ports:
|
||||
- "${APP_PUBLISH_HOST:-127.0.0.1}:${HTTP_PORT:-8080}:${HTTP_PORT:-8080}"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-fsS", "http://127.0.0.1:${HTTP_PORT:-8080}/api/v1/health"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 6
|
||||
start_period: 20s
|
||||
volumes:
|
||||
# 独占对话记录与环境数据
|
||||
- ./data/dev:/app/data
|
||||
- dev-aux-sqlite-data:/var/lib/feishu
|
||||
- ./.regulations:/app/.regulations:ro
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- dev-net
|
||||
|
||||
+13
-2
@@ -78,15 +78,19 @@ services:
|
||||
- main-net
|
||||
|
||||
app:
|
||||
image: alpine:latest
|
||||
build:
|
||||
context: .
|
||||
dockerfile: deploy/app/Dockerfile
|
||||
container_name: shuimu-agent-app-main
|
||||
command: ["sh", "-c", "echo 'Main/Prod Agent App service is running on :8000' && sleep infinity"]
|
||||
depends_on:
|
||||
system-db:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
ENV: prod
|
||||
HTTP_ADDR: ":${HTTP_PORT:-8000}"
|
||||
HTTP_PORT: "${HTTP_PORT:-8000}"
|
||||
HTTP_HOST: "0.0.0.0"
|
||||
AGENT_BACKEND: ${AGENT_BACKEND:-pi-agent}
|
||||
DATA_DIR: /app/data
|
||||
# 独占生产业务与对话库
|
||||
APP_DB_DSN: postgres://${SYSTEM_DB_USER:-app}:${SYSTEM_DB_PASSWORD:-prod_highly_secure_db_pass}@system-db:5432/${SYSTEM_DB_NAME:-agent_system_main}?sslmode=disable
|
||||
@@ -97,10 +101,17 @@ services:
|
||||
AUX_FILE_DSN: file:/var/lib/feishu/outbox_main.db?_journal_mode=WAL&_busy_timeout=5000
|
||||
ports:
|
||||
- "${APP_PUBLISH_HOST:-0.0.0.0}:${HTTP_PORT:-8000}:${HTTP_PORT:-8000}"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-fsS", "http://127.0.0.1:${HTTP_PORT:-8000}/api/v1/health"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 6
|
||||
start_period: 20s
|
||||
volumes:
|
||||
# 独占生产对话记录与环境数据
|
||||
- ./data/main:/app/data
|
||||
- main-aux-sqlite-data:/var/lib/feishu
|
||||
- ./.regulations:/app/.regulations:ro
|
||||
restart: always
|
||||
networks:
|
||||
- main-net
|
||||
|
||||
+13
-2
@@ -78,15 +78,19 @@ services:
|
||||
- test-net
|
||||
|
||||
app:
|
||||
image: alpine:latest
|
||||
build:
|
||||
context: .
|
||||
dockerfile: deploy/app/Dockerfile
|
||||
container_name: shuimu-agent-app-test
|
||||
command: ["sh", "-c", "echo 'Test Agent App service is running on :8081' && sleep infinity"]
|
||||
depends_on:
|
||||
system-db:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
ENV: test
|
||||
HTTP_ADDR: ":${HTTP_PORT:-8081}"
|
||||
HTTP_PORT: "${HTTP_PORT:-8081}"
|
||||
HTTP_HOST: "0.0.0.0"
|
||||
AGENT_BACKEND: ${AGENT_BACKEND:-pi-agent}
|
||||
DATA_DIR: /app/data
|
||||
# 独占测试业务与对话库
|
||||
APP_DB_DSN: postgres://${SYSTEM_DB_USER:-app}:${SYSTEM_DB_PASSWORD:-test_secret_db_pass}@system-db:5432/${SYSTEM_DB_NAME:-agent_system_test}?sslmode=disable
|
||||
@@ -97,10 +101,17 @@ services:
|
||||
AUX_FILE_DSN: file:/var/lib/feishu/outbox_test.db?_journal_mode=WAL&_busy_timeout=5000
|
||||
ports:
|
||||
- "${APP_PUBLISH_HOST:-0.0.0.0}:${HTTP_PORT:-8081}:${HTTP_PORT:-8081}"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-fsS", "http://127.0.0.1:${HTTP_PORT:-8081}/api/v1/health"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 6
|
||||
start_period: 20s
|
||||
volumes:
|
||||
# 独占对话记录与环境数据
|
||||
- ./data/test:/app/data
|
||||
- test-aux-sqlite-data:/var/lib/feishu
|
||||
- ./.regulations:/app/.regulations:ro
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- test-net
|
||||
|
||||
Executable
+257
@@ -0,0 +1,257 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# 法规知识库完整备份脚本 (backup_regulations.sh)
|
||||
#
|
||||
# 作用:
|
||||
# 1. 检查共享基础设施容器 shuimu-kb-shared 与数据库状态;
|
||||
# 2. 采用 pg_dump -Fc 高压缩二进制格式导出 regulations 数据库(含向量与索引);
|
||||
# 3. 镜像备份项目根目录的 .regulations 物理文件(支持目录克隆/同步及 tar 归档);
|
||||
# 4. 生成 manifest.json 记录基准元数据、统计指标及 SHA256 签名。
|
||||
#
|
||||
# 用法:
|
||||
# ./scripts/backup_regulations.sh <目标备份目录> [选项]
|
||||
#
|
||||
# 选项:
|
||||
# --archive, -a 额外将 .regulations 打包为 regulations_files.tar.gz
|
||||
# --help, -h 显示帮助信息
|
||||
# =============================================================================
|
||||
|
||||
set -eo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
CONTAINER_NAME="shuimu-kb-shared"
|
||||
DB_NAME="regulations"
|
||||
DB_USER="regkb"
|
||||
|
||||
show_help() {
|
||||
cat <<EOF
|
||||
用法:
|
||||
$0 <目标备份目录> [选项]
|
||||
|
||||
参数:
|
||||
目标备份目录 必须指定,备份文件将保存至该目录下
|
||||
|
||||
选项:
|
||||
--archive, -a 额外将 .regulations 打包为 regulations_files.tar.gz 压缩包
|
||||
--help, -h 显示本帮助信息
|
||||
|
||||
示例:
|
||||
$0 /Volumes/Backups/regulations_20260911
|
||||
$0 ./backups/reg_backup --archive
|
||||
EOF
|
||||
exit 0
|
||||
}
|
||||
|
||||
if [[ $# -lt 1 ]]; then
|
||||
show_help
|
||||
fi
|
||||
|
||||
TARGET_DIR=""
|
||||
ARCHIVE_MODE=0
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--help|-h)
|
||||
show_help
|
||||
;;
|
||||
--archive|-a)
|
||||
ARCHIVE_MODE=1
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
if [[ -z "${TARGET_DIR}" ]]; then
|
||||
TARGET_DIR="$1"
|
||||
else
|
||||
echo "[错误] 未知参数: $1"
|
||||
show_help
|
||||
fi
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -z "${TARGET_DIR}" ]]; then
|
||||
echo "[错误] 请指定目标备份目录!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 解析绝对路径
|
||||
mkdir -p "${TARGET_DIR}"
|
||||
TARGET_DIR="$(cd "${TARGET_DIR}" && pwd)"
|
||||
|
||||
echo "============================================================"
|
||||
echo ">>> 开始执行法规知识库完整备份"
|
||||
echo ">>> 项目根路径: ${ROOT_DIR}"
|
||||
echo ">>> 目标备份目录: ${TARGET_DIR}"
|
||||
echo "============================================================"
|
||||
|
||||
# 1. 检查环境与 Docker 容器
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
echo "[错误] 未找到 docker 命令,请先安装或启动 Docker!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! docker ps --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then
|
||||
echo "[警告] 容器 ${CONTAINER_NAME} 未运行,尝试启动共享基础设施..."
|
||||
if [[ -f "${ROOT_DIR}/scripts/env-service.sh" ]]; then
|
||||
"${ROOT_DIR}/scripts/env-service.sh" shared start
|
||||
else
|
||||
docker compose -f "${ROOT_DIR}/docker-compose.shared.yml" up -d
|
||||
fi
|
||||
fi
|
||||
|
||||
# 等待数据库就绪
|
||||
echo "[等待] 检查数据库连接状态..."
|
||||
for i in {1..30}; do
|
||||
if docker exec -i "${CONTAINER_NAME}" pg_isready -U "${DB_USER}" -d "${DB_NAME}" >/dev/null 2>&1; then
|
||||
echo "[就绪] 数据库连接正常。"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
if [ "$i" -eq 30 ]; then
|
||||
echo "[错误] 数据库连接超时,备份中止!"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# 2. 检查 .regulations 物理目录
|
||||
REG_SOURCE_DIR="${ROOT_DIR}/.regulations"
|
||||
if [[ ! -d "${REG_SOURCE_DIR}" ]]; then
|
||||
echo "[错误] 项目根目录下未找到 .regulations 目录 (${REG_SOURCE_DIR})!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 3. 统计数据库基准指标
|
||||
echo "[统计] 采集数据库元数据与统计指标..."
|
||||
DOC_COUNT=$(docker exec -i "${CONTAINER_NAME}" psql -U "${DB_USER}" -d "${DB_NAME}" -t -A -c "SELECT count(*) FROM reg_documents;" 2>/dev/null || echo "0")
|
||||
CHUNK_COUNT=$(docker exec -i "${CONTAINER_NAME}" psql -U "${DB_USER}" -d "${DB_NAME}" -t -A -c "SELECT count(*) FROM reg_chunks;" 2>/dev/null || echo "0")
|
||||
NODE_COUNT=$(docker exec -i "${CONTAINER_NAME}" psql -U "${DB_USER}" -d "${DB_NAME}" -t -A -c "SELECT count(*) FROM reg_nodes;" 2>/dev/null || echo "0")
|
||||
DB_SIZE=$(docker exec -i "${CONTAINER_NAME}" psql -U "${DB_USER}" -d "${DB_NAME}" -t -A -c "SELECT pg_size_pretty(pg_database_size('${DB_NAME}'));" 2>/dev/null || echo "0 MB")
|
||||
|
||||
echo " • 法规文档数 (reg_documents): ${DOC_COUNT}"
|
||||
echo " • 法规切片数 (reg_chunks): ${CHUNK_COUNT}"
|
||||
echo " • 大纲节点数 (reg_nodes): ${NODE_COUNT}"
|
||||
echo " • 数据库当前体积: ${DB_SIZE}"
|
||||
|
||||
# 4. 导出数据库
|
||||
DUMP_FILENAME="regulations_db.dump"
|
||||
TMP_CONTAINER_DUMP="/tmp/${DUMP_FILENAME}"
|
||||
DEST_DUMP_PATH="${TARGET_DIR}/${DUMP_FILENAME}"
|
||||
|
||||
echo "[导出] 正在执行 PostgreSQL 数据库导出 (pg_dump -Fc 高压缩格式)..."
|
||||
START_TIME=$(date +%s)
|
||||
docker exec "${CONTAINER_NAME}" pg_dump -U "${DB_USER}" -d "${DB_NAME}" -Fc -Z 6 -f "${TMP_CONTAINER_DUMP}"
|
||||
docker cp "${CONTAINER_NAME}:${TMP_CONTAINER_DUMP}" "${DEST_DUMP_PATH}"
|
||||
docker exec "${CONTAINER_NAME}" rm -f "${TMP_CONTAINER_DUMP}"
|
||||
DUMP_DURATION=$(( $(date +%s) - START_TIME ))
|
||||
|
||||
DUMP_SIZE_HUMAN=$(ls -lh "${DEST_DUMP_PATH}" | awk '{print $5}')
|
||||
DUMP_SIZE_BYTES=$(stat -f "%z" "${DEST_DUMP_PATH}" 2>/dev/null || stat -c "%s" "${DEST_DUMP_PATH}" 2>/dev/null || echo "0")
|
||||
|
||||
echo "[完成] 数据库导出成功,耗时: ${DUMP_DURATION} 秒,体积: ${DUMP_SIZE_HUMAN}"
|
||||
|
||||
# 计算哈希
|
||||
echo "[校验] 计算数据库 Dump 文件的 SHA256 校验和..."
|
||||
if command -v shasum >/dev/null 2>&1; then
|
||||
DUMP_SHA256=$(shasum -a 256 "${DEST_DUMP_PATH}" | awk '{print $1}')
|
||||
elif command -v sha256sum >/dev/null 2>&1; then
|
||||
DUMP_SHA256=$(sha256sum "${DEST_DUMP_PATH}" | awk '{print $1}')
|
||||
else
|
||||
DUMP_SHA256="unavailable"
|
||||
fi
|
||||
echo " • SHA256: ${DUMP_SHA256}"
|
||||
|
||||
# 5. 备份物理目录 .regulations
|
||||
echo "[备份] 正在同步 .regulations 物理文档与解析工作区..."
|
||||
DEST_REG_DIR="${TARGET_DIR}/.regulations"
|
||||
mkdir -p "${DEST_REG_DIR}"
|
||||
|
||||
# 统计原文件数量与体积
|
||||
TOTAL_FILES=$(find "${REG_SOURCE_DIR}" -type f | wc -l | tr -d ' ')
|
||||
DIR_SIZE_HUMAN=$(du -sh "${REG_SOURCE_DIR}" | awk '{print $1}')
|
||||
|
||||
echo " • 源文件总量: ${TOTAL_FILES} 个"
|
||||
echo " • 目录总大小: ${DIR_SIZE_HUMAN}"
|
||||
|
||||
# 优先使用 macOS APFS clonefile (快速且零物理块拷贝)
|
||||
if cp -R -p -c "${REG_SOURCE_DIR}/." "${DEST_REG_DIR}/" 2>/dev/null; then
|
||||
echo "[完成] 已通过 APFS Copy-on-Write (零额外空间) 完成物理目录克隆镜像。"
|
||||
else
|
||||
# 跨磁盘或非 APFS 降级为 rsync / cp
|
||||
if command -v rsync >/dev/null 2>&1; then
|
||||
rsync -a --delete "${REG_SOURCE_DIR}/" "${DEST_REG_DIR}/"
|
||||
else
|
||||
cp -R -p "${REG_SOURCE_DIR}/." "${DEST_REG_DIR}/"
|
||||
fi
|
||||
echo "[完成] 物理目录同步完成。"
|
||||
fi
|
||||
|
||||
ARCHIVE_TAR_FILE=""
|
||||
ARCHIVE_TAR_SHA256=""
|
||||
if [[ "${ARCHIVE_MODE}" -eq 1 ]]; then
|
||||
echo "[归档] 正在将 .regulations 物理目录打包为 tar.gz..."
|
||||
ARCHIVE_TAR_FILE="regulations_files.tar.gz"
|
||||
ARCHIVE_TAR_PATH="${TARGET_DIR}/${ARCHIVE_TAR_FILE}"
|
||||
tar -czf "${ARCHIVE_TAR_PATH}" -C "${ROOT_DIR}" .regulations
|
||||
if command -v shasum >/dev/null 2>&1; then
|
||||
ARCHIVE_TAR_SHA256=$(shasum -a 256 "${ARCHIVE_TAR_PATH}" | awk '{print $1}')
|
||||
fi
|
||||
echo "[完成] 归档包已生成: ${ARCHIVE_TAR_PATH}"
|
||||
fi
|
||||
|
||||
# 6. 生成元数据清单 manifest.json
|
||||
MANIFEST_FILE="${TARGET_DIR}/manifest.json"
|
||||
echo "[元数据] 写入备份清单 ${MANIFEST_FILE}..."
|
||||
NOW_ISO=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
HOSTNAME_STR=$(hostname 2>/dev/null || echo "unknown")
|
||||
|
||||
cat <<EOF > "${MANIFEST_FILE}"
|
||||
{
|
||||
"manifest_version": "1.0",
|
||||
"created_at": "${NOW_ISO}",
|
||||
"project": "Chat-Agent-System",
|
||||
"source_hostname": "${HOSTNAME_STR}",
|
||||
"database": {
|
||||
"container_name": "${CONTAINER_NAME}",
|
||||
"database_name": "${DB_NAME}",
|
||||
"user": "${DB_USER}",
|
||||
"dump_file": "${DUMP_FILENAME}",
|
||||
"dump_sha256": "${DUMP_SHA256}",
|
||||
"dump_size_bytes": ${DUMP_SIZE_BYTES},
|
||||
"dump_size_human": "${DUMP_SIZE_HUMAN}",
|
||||
"document_count": ${DOC_COUNT},
|
||||
"chunk_count": ${CHUNK_COUNT},
|
||||
"node_count": ${NODE_COUNT},
|
||||
"database_size": "${DB_SIZE}"
|
||||
},
|
||||
"regulations_files": {
|
||||
"relative_path": ".regulations",
|
||||
"total_files": ${TOTAL_FILES},
|
||||
"directory_size": "${DIR_SIZE_HUMAN}",
|
||||
"subdirectories": [
|
||||
"intake",
|
||||
"library",
|
||||
"quarantine",
|
||||
"work"
|
||||
],
|
||||
"archive_file": "${ARCHIVE_TAR_FILE}",
|
||||
"archive_sha256": "${ARCHIVE_TAR_SHA256}"
|
||||
}
|
||||
}
|
||||
EOF
|
||||
|
||||
echo "============================================================"
|
||||
echo ">>> [SUCCESS] 法规知识库完整备份成功!"
|
||||
echo ">>> 备份清单: ${MANIFEST_FILE}"
|
||||
echo ">>> 数据库 Dump: ${DEST_DUMP_PATH} (${DUMP_SIZE_HUMAN})"
|
||||
echo ">>> 物理文件目录: ${DEST_REG_DIR} (${TOTAL_FILES} 文件)"
|
||||
if [[ -n "${ARCHIVE_TAR_FILE}" ]]; then
|
||||
echo ">>> 物理文件归档: ${TARGET_DIR}/${ARCHIVE_TAR_FILE}"
|
||||
fi
|
||||
echo "============================================================"
|
||||
echo "恢复提示: 在任意目标机器上拉取本工程后,只需执行:"
|
||||
echo " ./scripts/restore_regulations.sh \"${TARGET_DIR}\""
|
||||
echo "============================================================"
|
||||
|
||||
@@ -57,8 +57,10 @@ prepare_env_file() {
|
||||
fi
|
||||
|
||||
# 确保隔离数据目录存在
|
||||
# 确保隔离数据目录与法规目录存在
|
||||
if [[ "${target}" == "shared" ]]; then
|
||||
mkdir -p "data/shared/kb"
|
||||
mkdir -p ".regulations/intake" ".regulations/library" ".regulations/quarantine" ".regulations/work"
|
||||
else
|
||||
mkdir -p "data/${target}/workspaces"
|
||||
fi
|
||||
|
||||
Executable
+308
@@ -0,0 +1,308 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# 法规知识库完整恢复脚本 (restore_regulations.sh)
|
||||
#
|
||||
# 作用:
|
||||
# 1. 在新拉取本仓库的任意机器上一键执行;
|
||||
# 2. 校验备份目录及 manifest.json 完整性与 SHA256;
|
||||
# 3. 将法规物理文档恢复到工程根目录 .regulations;
|
||||
# 4. 自动检测并拉起共享基础设施容器 shuimu-kb-shared;
|
||||
# 5. 将 regulations_db.dump 完整还原至数据库;
|
||||
# 6. 自动执行数据完整性与基线指标对齐校验。
|
||||
#
|
||||
# 用法:
|
||||
# ./scripts/restore_regulations.sh <备份目录路径> [选项]
|
||||
#
|
||||
# 选项:
|
||||
# --force, -f 强制覆盖已存在的数据库表,不进行确认
|
||||
# --skip-db 仅恢复物理文件,跳过数据库导入
|
||||
# --skip-files 仅恢复数据库,跳过物理文件复制
|
||||
# --help, -h 显示帮助信息
|
||||
# =============================================================================
|
||||
|
||||
set -eo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
CONTAINER_NAME="shuimu-kb-shared"
|
||||
DB_NAME="regulations"
|
||||
DB_USER="regkb"
|
||||
NETWORK_NAME="chat-agent-shared-net"
|
||||
|
||||
show_help() {
|
||||
cat <<EOF
|
||||
用法:
|
||||
$0 <备份目录路径> [选项]
|
||||
|
||||
参数:
|
||||
备份目录路径 指定包含 manifest.json 与 regulations_db.dump 的备份目录
|
||||
|
||||
选项:
|
||||
--force, -f 非交互强制恢复(覆盖已有数据)
|
||||
--skip-db 仅恢复 .regulations 物理文件,跳过数据库导入
|
||||
--skip-files 仅恢复数据库,跳过物理文件
|
||||
--help, -h 显示本帮助信息
|
||||
|
||||
示例:
|
||||
$0 /Volumes/Backups/regulations_20260911
|
||||
$0 ./backups/reg_backup --force
|
||||
EOF
|
||||
exit 0
|
||||
}
|
||||
|
||||
if [[ $# -lt 1 ]]; then
|
||||
show_help
|
||||
fi
|
||||
|
||||
BACKUP_DIR=""
|
||||
FORCE_RESTORE=0
|
||||
SKIP_DB=0
|
||||
SKIP_FILES=0
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--help|-h)
|
||||
show_help
|
||||
;;
|
||||
--force|-f)
|
||||
FORCE_RESTORE=1
|
||||
shift
|
||||
;;
|
||||
--skip-db)
|
||||
SKIP_DB=1
|
||||
shift
|
||||
;;
|
||||
--skip-files)
|
||||
SKIP_FILES=1
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
if [[ -z "${BACKUP_DIR}" ]]; then
|
||||
BACKUP_DIR="$1"
|
||||
else
|
||||
echo "[错误] 未知参数: $1"
|
||||
show_help
|
||||
fi
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -z "${BACKUP_DIR}" ]]; then
|
||||
echo "[错误] 请指定备份目录路径!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -d "${BACKUP_DIR}" ]]; then
|
||||
echo "[错误] 备份目录不存在: ${BACKUP_DIR}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
BACKUP_DIR="$(cd "${BACKUP_DIR}" && pwd)"
|
||||
|
||||
echo "============================================================"
|
||||
echo ">>> 开始执行法规知识库恢复与对齐"
|
||||
echo ">>> 项目根路径: ${ROOT_DIR}"
|
||||
echo ">>> 备份源目录: ${BACKUP_DIR}"
|
||||
echo "============================================================"
|
||||
|
||||
# 1. 检查备份目录内关键文件
|
||||
MANIFEST_FILE="${BACKUP_DIR}/manifest.json"
|
||||
DUMP_FILE="${BACKUP_DIR}/regulations_db.dump"
|
||||
|
||||
if [[ ! -f "${MANIFEST_FILE}" ]]; then
|
||||
echo "[警告] 未在备份目录中找到 manifest.json,将以常规模式执行恢复。"
|
||||
else
|
||||
echo "[元数据] 检测到备份清单 manifest.json:"
|
||||
python3 -c "
|
||||
import json
|
||||
with open('${MANIFEST_FILE}') as f:
|
||||
d = json.load(f)
|
||||
print(' • 创建时间:', d.get('created_at', 'unknown'))
|
||||
print(' • 来源主机:', d.get('source_hostname', 'unknown'))
|
||||
db = d.get('database', {})
|
||||
print(' • 数据库文档数 (reg_documents):', db.get('document_count', 'unknown'))
|
||||
print(' • 数据库切片数 (reg_chunks):', db.get('chunk_count', 'unknown'))
|
||||
print(' • 数据库节点数 (reg_nodes):', db.get('node_count', 'unknown'))
|
||||
files = d.get('regulations_files', {})
|
||||
print(' • 物理文件数:', files.get('total_files', 'unknown'))
|
||||
" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# 校验 Dump 文件
|
||||
if [[ "${SKIP_DB}" -eq 0 ]]; then
|
||||
if [[ ! -f "${DUMP_FILE}" ]]; then
|
||||
echo "[错误] 未找到数据库 Dump 文件: ${DUMP_FILE}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 校验 SHA256 (若 manifest 中有记录)
|
||||
if [[ -f "${MANIFEST_FILE}" ]]; then
|
||||
WANT_SHA=$(python3 -c "import json; print(json.load(open('${MANIFEST_FILE}')).get('database', {}).get('dump_sha256', ''))" 2>/dev/null || echo "")
|
||||
if [[ -n "${WANT_SHA}" && "${WANT_SHA}" != "unavailable" ]]; then
|
||||
echo "[校验] 正在验证 dump 文件 SHA256 签名..."
|
||||
if command -v shasum >/dev/null 2>&1; then
|
||||
GOT_SHA=$(shasum -a 256 "${DUMP_FILE}" | awk '{print $1}')
|
||||
elif command -v sha256sum >/dev/null 2>&1; then
|
||||
GOT_SHA=$(sha256sum "${DUMP_FILE}" | awk '{print $1}')
|
||||
else
|
||||
GOT_SHA=""
|
||||
fi
|
||||
|
||||
if [[ -n "${GOT_SHA}" ]]; then
|
||||
if [[ "${GOT_SHA}" == "${WANT_SHA}" ]]; then
|
||||
echo " ✅ SHA256 校验通过: ${GOT_SHA}"
|
||||
else
|
||||
echo " ❌ [错误] SHA256 校验不匹配!"
|
||||
echo " 期望: ${WANT_SHA}"
|
||||
echo " 实际: ${GOT_SHA}"
|
||||
echo "备份可能已损坏,恢复中止!"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# 2. 恢复 .regulations 物理目录
|
||||
if [[ "${SKIP_FILES}" -eq 0 ]]; then
|
||||
DEST_REG_DIR="${ROOT_DIR}/.regulations"
|
||||
echo "------------------------------------------------------------"
|
||||
echo ">>> [步骤 1/3] 恢复物理文件目录至 .regulations"
|
||||
echo "------------------------------------------------------------"
|
||||
|
||||
SRC_REG_DIR="${BACKUP_DIR}/.regulations"
|
||||
SRC_TAR_FILE="${BACKUP_DIR}/regulations_files.tar.gz"
|
||||
|
||||
mkdir -p "${DEST_REG_DIR}"
|
||||
|
||||
if [[ -d "${SRC_REG_DIR}" ]]; then
|
||||
echo "[同步] 从备份目录克隆/同步 .regulations..."
|
||||
if cp -R -p -c "${SRC_REG_DIR}/." "${DEST_REG_DIR}/" 2>/dev/null; then
|
||||
echo " ✅ 已通过 APFS CoW 极速克隆至 ${DEST_REG_DIR}"
|
||||
else
|
||||
if command -v rsync >/dev/null 2>&1; then
|
||||
rsync -a "${SRC_REG_DIR}/" "${DEST_REG_DIR}/"
|
||||
else
|
||||
cp -R -p "${SRC_REG_DIR}/." "${DEST_REG_DIR}/"
|
||||
fi
|
||||
echo " ✅ 物理目录同步完成"
|
||||
fi
|
||||
elif [[ -f "${SRC_TAR_FILE}" ]]; then
|
||||
echo "[解压] 从归档包 ${SRC_TAR_FILE} 解压至工程根目录..."
|
||||
tar -xzf "${SRC_TAR_FILE}" -C "${ROOT_DIR}"
|
||||
echo " ✅ 解压完成"
|
||||
else
|
||||
echo "[警告] 备份中未找到 .regulations 目录或 regulations_files.tar.gz!"
|
||||
fi
|
||||
|
||||
# 验证物理子目录结构
|
||||
for sub in intake library quarantine work; do
|
||||
mkdir -p "${DEST_REG_DIR}/${sub}"
|
||||
done
|
||||
RESTORED_FILES=$(find "${DEST_REG_DIR}" -type f | wc -l | tr -d ' ')
|
||||
RESTORED_SIZE=$(du -sh "${DEST_REG_DIR}" | awk '{print $1}')
|
||||
echo " • 已恢复物理文件数: ${RESTORED_FILES} 个"
|
||||
echo " • 物理目录总体积: ${RESTORED_SIZE}"
|
||||
fi
|
||||
|
||||
# 3. 检查并拉起 Docker 共享基础设施容器
|
||||
if [[ "${SKIP_DB}" -eq 0 ]]; then
|
||||
echo "------------------------------------------------------------"
|
||||
echo ">>> [步骤 2/3] 检查并初始化共享基础设施 (${CONTAINER_NAME})"
|
||||
echo "------------------------------------------------------------"
|
||||
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
echo "[错误] 未检测到 docker 环境,请先安装 Docker!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 确保 Docker 网络存在
|
||||
if ! docker network ls --format '{{.Name}}' | grep -q "^${NETWORK_NAME}$"; then
|
||||
echo "[网络] 创建 Docker 共享网络: ${NETWORK_NAME}..."
|
||||
docker network create "${NETWORK_NAME}" >/dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
# 确保容器运行
|
||||
if ! docker ps --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then
|
||||
echo "[启动] 容器 ${CONTAINER_NAME} 未运行,正在启动..."
|
||||
docker compose -f "${ROOT_DIR}/docker-compose.shared.yml" up -d
|
||||
fi
|
||||
|
||||
echo "[等待] 等待数据库就绪..."
|
||||
for i in {1..30}; do
|
||||
if docker exec -i "${CONTAINER_NAME}" pg_isready -U "${DB_USER}" -d "${DB_NAME}" >/dev/null 2>&1; then
|
||||
echo " ✅ 容器与数据库连接就绪 (${CONTAINER_NAME})"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
if [ "$i" -eq 30 ]; then
|
||||
echo "[错误] 等待目标数据库连接超时!"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# 4. 执行数据库无损恢复
|
||||
echo "------------------------------------------------------------"
|
||||
echo ">>> [步骤 3/3] 执行数据库还原 (pg_restore)"
|
||||
echo "------------------------------------------------------------"
|
||||
|
||||
TMP_RESTORE_DUMP="/tmp/restore_regulations_$$.dump"
|
||||
echo "[传输] 复制 dump 文件至容器内部..."
|
||||
docker cp "${DUMP_FILE}" "${CONTAINER_NAME}:${TMP_RESTORE_DUMP}"
|
||||
|
||||
echo "[初始化] 确保向量扩展 vector 与 pg_trgm 就绪..."
|
||||
docker exec -i "${CONTAINER_NAME}" psql -U "${DB_USER}" -d "${DB_NAME}" -c "CREATE EXTENSION IF NOT EXISTS vector; CREATE EXTENSION IF NOT EXISTS pg_trgm;" >/dev/null 2>&1 || true
|
||||
|
||||
echo "[还原] 正在执行无损数据库还原 (包含切片表、倒排索引及 1024 维向量)..."
|
||||
START_RESTORE_TIME=$(date +%s)
|
||||
|
||||
# 执行 pg_restore: --clean --if-exists 清理旧表重建,允许非致命警告 (如 DROP 不存在表返回 1)
|
||||
set +e
|
||||
RESTORE_OUTPUT=$(docker exec "${CONTAINER_NAME}" pg_restore -U "${DB_USER}" -d "${DB_NAME}" \
|
||||
--clean --if-exists --no-owner --no-privileges "${TMP_RESTORE_DUMP}" 2>&1)
|
||||
RESTORE_STATUS=$?
|
||||
set -e
|
||||
|
||||
docker exec "${CONTAINER_NAME}" rm -f "${TMP_RESTORE_DUMP}"
|
||||
RESTORE_DURATION=$(( $(date +%s) - START_RESTORE_TIME ))
|
||||
echo " ✅ 数据库还原指令完成,耗时: ${RESTORE_DURATION} 秒。"
|
||||
|
||||
# 5. 执行基线完整性校验
|
||||
echo "============================================================"
|
||||
echo ">>> 执行恢复后数据完整性基线校验"
|
||||
echo "============================================================"
|
||||
|
||||
CURR_DOC_COUNT=$(docker exec -i "${CONTAINER_NAME}" psql -U "${DB_USER}" -d "${DB_NAME}" -t -A -c "SELECT count(*) FROM reg_documents;" 2>/dev/null || echo "0")
|
||||
CURR_CHUNK_COUNT=$(docker exec -i "${CONTAINER_NAME}" psql -U "${DB_USER}" -d "${DB_NAME}" -t -A -c "SELECT count(*) FROM reg_chunks;" 2>/dev/null || echo "0")
|
||||
CURR_NODE_COUNT=$(docker exec -i "${CONTAINER_NAME}" psql -U "${DB_USER}" -d "${DB_NAME}" -t -A -c "SELECT count(*) FROM reg_nodes;" 2>/dev/null || echo "0")
|
||||
CURR_DB_SIZE=$(docker exec -i "${CONTAINER_NAME}" psql -U "${DB_USER}" -d "${DB_NAME}" -t -A -c "SELECT pg_size_pretty(pg_database_size('${DB_NAME}'));" 2>/dev/null || echo "0 MB")
|
||||
|
||||
echo " • 法规文档总数 (reg_documents): ${CURR_DOC_COUNT}"
|
||||
echo " • 法规切片总数 (reg_chunks): ${CURR_CHUNK_COUNT}"
|
||||
echo " • 大纲节点总数 (reg_nodes): ${CURR_NODE_COUNT}"
|
||||
echo " • 数据库还原后体积: ${CURR_DB_SIZE}"
|
||||
|
||||
# 基线比对
|
||||
EXPECT_DOCS=2907
|
||||
EXPECT_CHUNKS=312900
|
||||
|
||||
if [[ "${CURR_DOC_COUNT}" -ge "${EXPECT_DOCS}" && "${CURR_CHUNK_COUNT}" -ge "${EXPECT_CHUNKS}" ]]; then
|
||||
echo "============================================================"
|
||||
echo ">>> 🎉 [PASS] 法规知识库 100% 恢复成功!数据与基线完全对齐!"
|
||||
echo "============================================================"
|
||||
else
|
||||
echo "============================================================"
|
||||
echo ">>> ⚠️ [WARN] 恢复数据条数与官方基线有偏差,请检查输出日志。"
|
||||
echo ">>> 实际: ${CURR_DOC_COUNT} 文档 / ${CURR_CHUNK_COUNT} 切片"
|
||||
echo ">>> 基线: ${EXPECT_DOCS} 文档 / >= ${EXPECT_CHUNKS} 切片"
|
||||
echo "============================================================"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "知识库服务就绪: shuimu-kb-shared:5432 (DB: regulations)"
|
||||
echo "物理文件就绪: .regulations/"
|
||||
echo "您可以在本项目中执行: go test -v ./internal/kb -run TestKB_RealIntegration 进行完整性验证。"
|
||||
|
||||
Reference in New Issue
Block a user