Add the real PostgreSQL and Redis CI gate (make test-backend-pg, test-pg job)
Why: the approved collaboration plan U2 requires CI to run the backend suite against real PostgreSQL+pgvector and Redis and fail when the services are missing, instead of silently skipping the tier. Evidence: local gate green twice against PostgreSQL 18.6+pgvector and Redis 8.10.2 (isolated svc.sh instance); the target fails fast when AGENT_PG_TEST_DSN/AGENT_REDIS_TEST_ADDR are unset. Impact: new make target test-backend-pg (env-guarded) and a required test-pg job (apt postgresql-16-pgvector + redis-server, ephemeral per-run services); existing test job unchanged. Refs: proposals/2026-10-11-collaboration-storage-and-distribution.md U2; docs/guides/team-storage.md section 5.
This commit is contained in:
@@ -174,3 +174,192 @@ jobs:
|
||||
run: make test
|
||||
- name: Build the single binary
|
||||
run: make build
|
||||
|
||||
# 协作存储方案 U2：真实 PostgreSQL + pgvector + Redis 门禁。
|
||||
# 缺服务时失败而非跳过（make test-backend-pg 硬校验环境变量）；
|
||||
# 组件版本随 job 输出固定，供验收记录对齐（方案 KTD12/U2）。
|
||||
test-pg:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
GOPROXY: https://goproxy.cn,direct
|
||||
steps:
|
||||
- name: Checkout
|
||||
env:
|
||||
TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
SUBMODULE_TOKEN: ${{ secrets.SUBMODULE_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Credential handling stays inline: no file from the candidate is executed.
|
||||
# Workflow provenance and runner isolation still require platform controls.
|
||||
python3 - <<'PY_CHECKOUT'
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
# 服务端基址取运行器注入的 GITHUB_SERVER_URL（平台控制、候选 PR 不可影响）。
|
||||
# 2026-10-05 修复（run 948/949 实测）：注入值为内网直连形态
|
||||
# 'http://172.17.0.1:3000'（docker 网关 → 宿主 Gitea），与公网拼写不同且为
|
||||
# http——此前硬编码常量严格比对会拒绝合法检出。校验收宽为「http(s) + 非空
|
||||
# netloc」；凭据仍只附着于「该基址 + 下方仓库白名单」构造的 URL。
|
||||
SERVER = os.environ.get("GITHUB_SERVER_URL", "").rstrip("/")
|
||||
parsed_server = urlsplit(SERVER)
|
||||
if parsed_server.scheme not in {"http", "https"} or not parsed_server.netloc:
|
||||
sys.exit("Checkout server must be the platform-injected http(s) URL")
|
||||
SSH = "ssh://git@git-ssh.u1680832.nyat.app:31411/weichen/"
|
||||
# Reviewed allowlist, independent of candidate .gitmodules. Add repositories
|
||||
# here only with an explicit review of the checkout credential's scope.
|
||||
REPOSITORIES = {
|
||||
"modules/core": "Agent-System-Core",
|
||||
"modules/backend": "Agent-System-Backend",
|
||||
"modules/frontend": "Agent-System-Frontend",
|
||||
"modules/ui": "Agent-System-UI",
|
||||
"modules/documents": "Agent-System-Documents",
|
||||
"deps/agent-system-core": "Agent-System-Core",
|
||||
"deps/development-guidelines": "Development-Guidelines",
|
||||
"deps/agent-native": "Agent-Native",
|
||||
"deps/agent-squad": "Agent-Squad",
|
||||
"deps/agent-intent": "Agent-Intent",
|
||||
"deps/agent-command": "Agent-Command",
|
||||
"deps/error-code-system": "Error-Code-System",
|
||||
"deps/logging-system": "Logging-System",
|
||||
}
|
||||
root_token = os.environ.pop("TOKEN", "")
|
||||
component_token = os.environ.pop("SUBMODULE_TOKEN", "")
|
||||
if not root_token or not component_token:
|
||||
sys.exit("Checkout requires TOKEN and SUBMODULE_TOKEN")
|
||||
if os.environ.get("GITHUB_REPOSITORY") != "weichen/Agent-System-Dev":
|
||||
sys.exit("Checkout repository is outside the reviewed allowlist")
|
||||
sha = os.environ.get("GITHUB_SHA", "")
|
||||
if not re.fullmatch(r"[0-9a-fA-F]{40}", sha):
|
||||
sys.exit("Checkout requires a fixed commit SHA")
|
||||
|
||||
# Discard inherited Git tracing/config/SSH hooks. Auth lives only in one
|
||||
# Git process tree, in an exact URL-scoped header, never in a URL or file.
|
||||
clean_env = {k: v for k, v in os.environ.items() if not k.startswith("GIT_")}
|
||||
clean_env.update(GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_SYSTEM="/dev/null",
|
||||
GIT_CONFIG_GLOBAL="/dev/null", GIT_TERMINAL_PROMPT="0")
|
||||
config = {
|
||||
"core.hooksPath": "/dev/null", "core.fsmonitor": "false",
|
||||
"init.templateDir": "", "credential.helper": "",
|
||||
"fetch.recurseSubmodules": "false", "submodule.recurse": "false",
|
||||
"http.followRedirects": "false", "protocol.allow": "never",
|
||||
"protocol." + urlsplit(SERVER).scheme + ".allow": "always",
|
||||
}
|
||||
|
||||
|
||||
def git(repo, *args, extra=None):
|
||||
values = config | (extra or {})
|
||||
env = clean_env | {"GIT_CONFIG_COUNT": str(len(values))}
|
||||
for index, (key, value) in enumerate(values.items()):
|
||||
env[f"GIT_CONFIG_KEY_{index}"] = key
|
||||
env[f"GIT_CONFIG_VALUE_{index}"] = value
|
||||
return subprocess.run(["git", "-C", str(repo), *args], env=env,
|
||||
check=True, stdout=subprocess.PIPE).stdout
|
||||
|
||||
|
||||
def materialize(repo, depth=0):
|
||||
if depth > 16:
|
||||
raise ValueError("Submodule depth exceeds the reviewed checkout limit")
|
||||
links = {}
|
||||
for entry in git(repo, "ls-tree", "-rz", "HEAD").split(b"\0"):
|
||||
if not entry:
|
||||
continue
|
||||
metadata, path = entry.split(b"\t", 1)
|
||||
mode, kind, commit = metadata.decode().split()
|
||||
if mode == "160000":
|
||||
links[path.decode()] = commit
|
||||
if not links:
|
||||
return
|
||||
raw = git(repo, "config", "--no-includes", "--blob", "HEAD:.gitmodules", "--null", "--list")
|
||||
entries = {}
|
||||
for item in raw.split(b"\0"):
|
||||
if not item:
|
||||
continue
|
||||
key, value = item.decode().split("\n", 1)
|
||||
match = re.fullmatch(r"submodule\.(.+)\.(path|url)", key)
|
||||
if not match:
|
||||
raise ValueError("Unsupported candidate .gitmodules setting")
|
||||
name, field = match.groups()
|
||||
if field in entries.setdefault(name, {}):
|
||||
raise ValueError("Duplicate candidate .gitmodules setting")
|
||||
entries[name][field] = value
|
||||
if set(entries) != set(links):
|
||||
raise ValueError("Submodule names must match the fixed gitlink paths")
|
||||
# Validate the entire level before any authenticated component fetch.
|
||||
for name, fields in entries.items():
|
||||
repository = REPOSITORIES.get(name)
|
||||
if not repository or fields.get("path") != name:
|
||||
raise ValueError("Submodule path is outside the reviewed allowlist")
|
||||
trusted_url = f"{SERVER}/weichen/{repository}.git"
|
||||
if fields.get("url") not in {SSH + repository + ".git", trusted_url}:
|
||||
raise ValueError("Submodule URL is outside the reviewed allowlist")
|
||||
for name in entries:
|
||||
url = f"{SERVER}/weichen/{REPOSITORIES[name]}.git"
|
||||
# Persist only the reviewed public URL so Git recognizes the
|
||||
# initialized submodule after process-scoped config is gone.
|
||||
git(repo, "config", f"submodule.{name}.url", url)
|
||||
git(repo, "submodule", "update", "--init", "--checkout", "--", name,
|
||||
extra={f"submodule.{name}.url": url,
|
||||
f"submodule.{name}.update": "checkout",
|
||||
f"http.{url}.extraheader": "AUTHORIZATION: token " + component_token})
|
||||
child = repo / name
|
||||
if git(child, "rev-parse", "HEAD").decode().strip() != links[name]:
|
||||
raise ValueError("Materialized submodule does not match the fixed SHA")
|
||||
materialize(child, depth + 1)
|
||||
|
||||
|
||||
try:
|
||||
root = Path.cwd()
|
||||
# An ephemeral clean checkout is required; do not reuse prior Git state.
|
||||
if (root / ".git").exists():
|
||||
raise ValueError("Checkout requires a fresh workspace")
|
||||
git(root, "init", "-q")
|
||||
url = SERVER + "/weichen/Agent-System-Dev.git"
|
||||
git(root, "remote", "add", "origin", url)
|
||||
git(root, "fetch", "--depth=1", "origin", sha,
|
||||
extra={f"http.{url}.extraheader": "AUTHORIZATION: token " + root_token})
|
||||
git(root, "checkout", "-q", "--detach", "FETCH_HEAD")
|
||||
materialize(root)
|
||||
except (ValueError, OSError, subprocess.CalledProcessError) as error:
|
||||
sys.exit(f"Checkout refused: {error}")
|
||||
print("Checkout complete; credentials were not persisted in Git configuration")
|
||||
PY_CHECKOUT
|
||||
unset TOKEN SUBMODULE_TOKEN
|
||||
- name: Install Go 1.27.1
|
||||
run: |
|
||||
set -euo pipefail
|
||||
curl -fsSL -o "$RUNNER_TEMP/go.tgz" https://mirrors.aliyun.com/golang/go1.27.1.linux-amd64.tar.gz
|
||||
tar -C /usr/local -xzf "$RUNNER_TEMP/go.tgz"
|
||||
ln -sfn /usr/local/go/bin/go /usr/local/bin/go
|
||||
ln -sfn /usr/local/go/bin/gofmt /usr/local/bin/gofmt
|
||||
- name: Install PostgreSQL, pgvector and Redis
|
||||
run: |
|
||||
set -euo pipefail
|
||||
SUDO=""
|
||||
if command -v sudo >/dev/null 2>&1 && [ [ "$(id -u)" != 0 ]; then SUDO=sudo; fi
|
||||
$SUDO apt-get update -y
|
||||
$SUDO apt-get install -y postgresql postgresql-contrib postgresql-16-pgvector redis-server
|
||||
# 固定组件/扩展版本输出（随日志归档，作为验收对齐依据）。
|
||||
psql --version
|
||||
$SUDO -u postgres psql -tAc "SELECT extversion FROM pg_extension WHERE extname='pgvector'" || true
|
||||
redis-server --version
|
||||
- name: Start PostgreSQL and Redis
|
||||
run: |
|
||||
set -euo pipefail
|
||||
SUDO=""
|
||||
if command -v sudo >/dev/null 2>&1 && [ "$(id -u)" != 0 ]; then SUDO=sudo; fi
|
||||
$SUDO service postgresql start || $SUDO pg_ctlcluster 16 main start
|
||||
$SUDO -u postgres psql -c "CREATE USER ci_agent PASSWORD 'ci_job_local_only' CREATEDB"
|
||||
$SUDO -u postgres psql -c "CREATE DATABASE agent_test OWNER ci_agent"
|
||||
# 预装 pg_trgm 到 public：与测试辅助的幂等预装一致，锁定的组件契约。
|
||||
$SUDO -u postgres psql -d agent_test -c "CREATE EXTENSION IF NOT EXISTS pg_trgm"
|
||||
redis-server --daemonize yes --port 6399 --save '' --appendonly no
|
||||
redis-cli -p 6399 ping
|
||||
- name: Backend tests against real PostgreSQL and Redis (required, no silent skip)
|
||||
env:
|
||||
AGENT_PG_TEST_DSN: postgres://ci_agent:ci_job_local_only@127.0.0.1:5432/agent_test?sslmode=disable
|
||||
AGENT_REDIS_TEST_ADDR: 127.0.0.1:6399
|
||||
run: make test-backend-pg
|
||||
|
||||
Reference in New Issue
Block a user