From 227c2a120ec0227d8cc590e7b9781f3a8b27d27b Mon Sep 17 00:00:00 2001 From: weichen Date: Sun, 11 Oct 2026 03:14:22 +0000 Subject: [PATCH] Add the real PostgreSQL and Redis CI gate (make test-backend-pg, test-pg job) Why: the approved collaboration plan U2 requires CI to run the backend suite against real PostgreSQL+pgvector and Redis and fail when the services are missing, instead of silently skipping the tier. Evidence: local gate green twice against PostgreSQL 18.6+pgvector and Redis 8.10.2 (isolated svc.sh instance); the target fails fast when AGENT_PG_TEST_DSN/AGENT_REDIS_TEST_ADDR are unset. Impact: new make target test-backend-pg (env-guarded) and a required test-pg job (apt postgresql-16-pgvector + redis-server, ephemeral per-run services); existing test job unchanged. Refs: proposals/2026-10-11-collaboration-storage-and-distribution.md U2; docs/guides/team-storage.md section 5. --- .gitea/workflows/ci.yaml | 189 +++++++++++++++++++++++++++++++++++++++ Makefile | 10 ++- 2 files changed, 198 insertions(+), 1 deletion(-) diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 1de4a457..73973f23 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -174,3 +174,192 @@ jobs: run: make test - name: Build the single binary run: make build + + # 协作存储方案 U2：真实 PostgreSQL + pgvector + Redis 门禁。 + # 缺服务时失败而非跳过（make test-backend-pg 硬校验环境变量）； + # 组件版本随 job 输出固定，供验收记录对齐（方案 KTD12/U2）。 + test-pg: + runs-on: ubuntu-latest + env: + GOPROXY: https://goproxy.cn,direct + steps: + - name: Checkout + env: + TOKEN: ${{ secrets.GITHUB_TOKEN }} + SUBMODULE_TOKEN: ${{ secrets.SUBMODULE_TOKEN }} + run: | + set -euo pipefail + # Credential handling stays inline: no file from the candidate is executed. + # Workflow provenance and runner isolation still require platform controls. + python3 - <<'PY_CHECKOUT' + import os + from pathlib import Path + import re + import subprocess + import sys + from urllib.parse import urlsplit + + # 服务端基址取运行器注入的 GITHUB_SERVER_URL（平台控制、候选 PR 不可影响）。 + # 2026-10-05 修复（run 948/949 实测）：注入值为内网直连形态 + # 'http://172.17.0.1:3000'（docker 网关 → 宿主 Gitea），与公网拼写不同且为 + # http——此前硬编码常量严格比对会拒绝合法检出。校验收宽为「http(s) + 非空 + # netloc」；凭据仍只附着于「该基址 + 下方仓库白名单」构造的 URL。 + SERVER = os.environ.get("GITHUB_SERVER_URL", "").rstrip("/") + parsed_server = urlsplit(SERVER) + if parsed_server.scheme not in {"http", "https"} or not parsed_server.netloc: + sys.exit("Checkout server must be the platform-injected http(s) URL") + SSH = "ssh://git@git-ssh.u1680832.nyat.app:31411/weichen/" + # Reviewed allowlist, independent of candidate .gitmodules. Add repositories + # here only with an explicit review of the checkout credential's scope. + REPOSITORIES = { + "modules/core": "Agent-System-Core", + "modules/backend": "Agent-System-Backend", + "modules/frontend": "Agent-System-Frontend", + "modules/ui": "Agent-System-UI", + "modules/documents": "Agent-System-Documents", + "deps/agent-system-core": "Agent-System-Core", + "deps/development-guidelines": "Development-Guidelines", + "deps/agent-native": "Agent-Native", + "deps/agent-squad": "Agent-Squad", + "deps/agent-intent": "Agent-Intent", + "deps/agent-command": "Agent-Command", + "deps/error-code-system": "Error-Code-System", + "deps/logging-system": "Logging-System", + } + root_token = os.environ.pop("TOKEN", "") + component_token = os.environ.pop("SUBMODULE_TOKEN", "") + if not root_token or not component_token: + sys.exit("Checkout requires TOKEN and SUBMODULE_TOKEN") + if os.environ.get("GITHUB_REPOSITORY") != "weichen/Agent-System-Dev": + sys.exit("Checkout repository is outside the reviewed allowlist") + sha = os.environ.get("GITHUB_SHA", "") + if not re.fullmatch(r"[0-9a-fA-F]{40}", sha): + sys.exit("Checkout requires a fixed commit SHA") + + # Discard inherited Git tracing/config/SSH hooks. Auth lives only in one + # Git process tree, in an exact URL-scoped header, never in a URL or file. + clean_env = {k: v for k, v in os.environ.items() if not k.startswith("GIT_")} + clean_env.update(GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_SYSTEM="/dev/null", + GIT_CONFIG_GLOBAL="/dev/null", GIT_TERMINAL_PROMPT="0") + config = { + "core.hooksPath": "/dev/null", "core.fsmonitor": "false", + "init.templateDir": "", "credential.helper": "", + "fetch.recurseSubmodules": "false", "submodule.recurse": "false", + "http.followRedirects": "false", "protocol.allow": "never", + "protocol." + urlsplit(SERVER).scheme + ".allow": "always", + } + + + def git(repo, *args, extra=None): + values = config | (extra or {}) + env = clean_env | {"GIT_CONFIG_COUNT": str(len(values))} + for index, (key, value) in enumerate(values.items()): + env[f"GIT_CONFIG_KEY_{index}"] = key + env[f"GIT_CONFIG_VALUE_{index}"] = value + return subprocess.run(["git", "-C", str(repo), *args], env=env, + check=True, stdout=subprocess.PIPE).stdout + + + def materialize(repo, depth=0): + if depth > 16: + raise ValueError("Submodule depth exceeds the reviewed checkout limit") + links = {} + for entry in git(repo, "ls-tree", "-rz", "HEAD").split(b"\0"): + if not entry: + continue + metadata, path = entry.split(b"\t", 1) + mode, kind, commit = metadata.decode().split() + if mode == "160000": + links[path.decode()] = commit + if not links: + return + raw = git(repo, "config", "--no-includes", "--blob", "HEAD:.gitmodules", "--null", "--list") + entries = {} + for item in raw.split(b"\0"): + if not item: + continue + key, value = item.decode().split("\n", 1) + match = re.fullmatch(r"submodule\.(.+)\.(path|url)", key) + if not match: + raise ValueError("Unsupported candidate .gitmodules setting") + name, field = match.groups() + if field in entries.setdefault(name, {}): + raise ValueError("Duplicate candidate .gitmodules setting") + entries[name][field] = value + if set(entries) != set(links): + raise ValueError("Submodule names must match the fixed gitlink paths") + # Validate the entire level before any authenticated component fetch. + for name, fields in entries.items(): + repository = REPOSITORIES.get(name) + if not repository or fields.get("path") != name: + raise ValueError("Submodule path is outside the reviewed allowlist") + trusted_url = f"{SERVER}/weichen/{repository}.git" + if fields.get("url") not in {SSH + repository + ".git", trusted_url}: + raise ValueError("Submodule URL is outside the reviewed allowlist") + for name in entries: + url = f"{SERVER}/weichen/{REPOSITORIES[name]}.git" + # Persist only the reviewed public URL so Git recognizes the + # initialized submodule after process-scoped config is gone. + git(repo, "config", f"submodule.{name}.url", url) + git(repo, "submodule", "update", "--init", "--checkout", "--", name, + extra={f"submodule.{name}.url": url, + f"submodule.{name}.update": "checkout", + f"http.{url}.extraheader": "AUTHORIZATION: token " + component_token}) + child = repo / name + if git(child, "rev-parse", "HEAD").decode().strip() != links[name]: + raise ValueError("Materialized submodule does not match the fixed SHA") + materialize(child, depth + 1) + + + try: + root = Path.cwd() + # An ephemeral clean checkout is required; do not reuse prior Git state. + if (root / ".git").exists(): + raise ValueError("Checkout requires a fresh workspace") + git(root, "init", "-q") + url = SERVER + "/weichen/Agent-System-Dev.git" + git(root, "remote", "add", "origin", url) + git(root, "fetch", "--depth=1", "origin", sha, + extra={f"http.{url}.extraheader": "AUTHORIZATION: token " + root_token}) + git(root, "checkout", "-q", "--detach", "FETCH_HEAD") + materialize(root) + except (ValueError, OSError, subprocess.CalledProcessError) as error: + sys.exit(f"Checkout refused: {error}") + print("Checkout complete; credentials were not persisted in Git configuration") + PY_CHECKOUT + unset TOKEN SUBMODULE_TOKEN + - name: Install Go 1.27.1 + run: | + set -euo pipefail + curl -fsSL -o "$RUNNER_TEMP/go.tgz" https://mirrors.aliyun.com/golang/go1.27.1.linux-amd64.tar.gz + tar -C /usr/local -xzf "$RUNNER_TEMP/go.tgz" + ln -sfn /usr/local/go/bin/go /usr/local/bin/go + ln -sfn /usr/local/go/bin/gofmt /usr/local/bin/gofmt + - name: Install PostgreSQL, pgvector and Redis + run: | + set -euo pipefail + SUDO="" + if command -v sudo >/dev/null 2>&1 && [ [ "$(id -u)" != 0 ]; then SUDO=sudo; fi + $SUDO apt-get update -y + $SUDO apt-get install -y postgresql postgresql-contrib postgresql-16-pgvector redis-server + # 固定组件/扩展版本输出（随日志归档，作为验收对齐依据）。 + psql --version + $SUDO -u postgres psql -tAc "SELECT extversion FROM pg_extension WHERE extname='pgvector'" || true + redis-server --version + - name: Start PostgreSQL and Redis + run: | + set -euo pipefail + SUDO="" + if command -v sudo >/dev/null 2>&1 && [ "$(id -u)" != 0 ]; then SUDO=sudo; fi + $SUDO service postgresql start || $SUDO pg_ctlcluster 16 main start + $SUDO -u postgres psql -c "CREATE USER ci_agent PASSWORD 'ci_job_local_only' CREATEDB" + $SUDO -u postgres psql -c "CREATE DATABASE agent_test OWNER ci_agent" + # 预装 pg_trgm 到 public：与测试辅助的幂等预装一致，锁定的组件契约。 + $SUDO -u postgres psql -d agent_test -c "CREATE EXTENSION IF NOT EXISTS pg_trgm" + redis-server --daemonize yes --port 6399 --save '' --appendonly no + redis-cli -p 6399 ping + - name: Backend tests against real PostgreSQL and Redis (required, no silent skip) + env: + AGENT_PG_TEST_DSN: postgres://ci_agent:ci_job_local_only@127.0.0.1:5432/agent_test?sslmode=disable + AGENT_REDIS_TEST_ADDR: 127.0.0.1:6399 + run: make test-backend-pg diff --git a/Makefile b/Makefile index 269c365c..74d4fa98 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ PLUGINS := plugins COMBO := core=$(shell git -C $(CORE) rev-parse --short HEAD 2>/dev/null)+backend=$(shell git -C $(BACKEND) rev-parse --short HEAD 2>/dev/null)+frontend=$(shell git -C $(FRONTEND) rev-parse --short HEAD 2>/dev/null)+ui=$(shell git -C $(UI) rev-parse --short HEAD 2>/dev/null)+agentcmd=$(shell git -C deps/agent-command rev-parse --short HEAD 2>/dev/null)+documents=$(shell git -C modules/documents rev-parse --short HEAD 2>/dev/null)+native=$(shell git -C deps/agent-native rev-parse --short HEAD 2>/dev/null)+squad=$(shell git -C deps/agent-squad rev-parse --short HEAD 2>/dev/null)+intent=$(shell git -C deps/agent-intent rev-parse --short HEAD 2>/dev/null) VERSION := $(shell git describe --tags --always --dirty 2>/dev/null)+$(COMBO) -.PHONY: help sync status build frontend svc svc-list svc-info svc-commits svc-history svc-update test-backend test-frontend test-ui test-core test-agentcmd test-pins test-libs dev pin clean setup precheck task-start task-sync task-close task-finish facts-verify evidence version +.PHONY: help sync status build frontend svc svc-list svc-info svc-commits svc-history svc-update test-backend test-backend-pg test-frontend test-ui test-core test-agentcmd test-pins test-libs dev pin clean setup precheck task-start task-sync task-close task-finish facts-verify evidence version version: @echo "agent-system $(VERSION)" @@ -101,6 +101,14 @@ test-core: test-backend: cd $(BACKEND) && go test ./... +# 真实 PG/Redis 门禁（协作存储方案 U2）：缺环境直接失败而非静默跳过。 +# CI 的必跑 job 提供原生服务；本地用 scripts/svc.sh db create <名> --port 8897 +# 与 redis-server --port 6399 起隔离实例（见 docs/guides/team-storage.md）。 +test-backend-pg: + @test -n "$${AGENT_PG_TEST_DSN}" || { echo "test-backend-pg: AGENT_PG_TEST_DSN 未设置（真实 PostgreSQL 必需，不静默跳过）"; exit 1; } + @test -n "$${AGENT_REDIS_TEST_ADDR}" || { echo "test-backend-pg: AGENT_REDIS_TEST_ADDR 未设置（真实 Redis 必需，不静默跳过）"; exit 1; } + cd $(BACKEND) && go test -count=1 ./... + eval-deliverables: cd $(BACKEND) && go test ./eval/ ./internal/quality/ -count=1