weichen
227c2a120e
Why: the approved collaboration plan U2 requires CI to run the backend suite against real PostgreSQL+pgvector and Redis and fail when the services are missing, instead of silently skipping the tier. Evidence: local gate green twice against PostgreSQL 18.6+pgvector and Redis 8.10.2 (isolated svc.sh instance); the target fails fast when AGENT_PG_TEST_DSN/AGENT_REDIS_TEST_ADDR are unset. Impact: new make target test-backend-pg (env-guarded) and a required test-pg job (apt postgresql-16-pgvector + redis-server, ephemeral per-run services); existing test job unchanged. Refs: proposals/2026-10-11-collaboration-storage-and-distribution.md U2; docs/guides/team-storage.md section 5.
366 lines
19 KiB
YAML
366 lines
19 KiB
YAML
name: ci
|
||
on:
|
||
pull_request:
|
||
branches: [main]
|
||
push:
|
||
branches: [main]
|
||
|
||
jobs:
|
||
test:
|
||
runs-on: ubuntu-latest
|
||
env:
|
||
GOPROXY: https://goproxy.cn,direct
|
||
steps:
|
||
- name: Checkout
|
||
env:
|
||
TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||
SUBMODULE_TOKEN: ${{ secrets.SUBMODULE_TOKEN }}
|
||
run: |
|
||
set -euo pipefail
|
||
# Credential handling stays inline: no file from the candidate is executed.
|
||
# Workflow provenance and runner isolation still require platform controls.
|
||
python3 - <<'PY_CHECKOUT'
|
||
import os
|
||
from pathlib import Path
|
||
import re
|
||
import subprocess
|
||
import sys
|
||
from urllib.parse import urlsplit
|
||
|
||
# 服务端基址取运行器注入的 GITHUB_SERVER_URL（平台控制、候选 PR 不可影响）。
|
||
# 2026-10-05 修复（run 948/949 实测）：注入值为内网直连形态
|
||
# 'http://172.17.0.1:3000'（docker 网关 → 宿主 Gitea），与公网拼写不同且为
|
||
# http——此前硬编码常量严格比对会拒绝合法检出。校验收宽为「http(s) + 非空
|
||
# netloc」；凭据仍只附着于「该基址 + 下方仓库白名单」构造的 URL。
|
||
SERVER = os.environ.get("GITHUB_SERVER_URL", "").rstrip("/")
|
||
parsed_server = urlsplit(SERVER)
|
||
if parsed_server.scheme not in {"http", "https"} or not parsed_server.netloc:
|
||
sys.exit("Checkout server must be the platform-injected http(s) URL")
|
||
SSH = "ssh://git@git-ssh.u1680832.nyat.app:31411/weichen/"
|
||
# Reviewed allowlist, independent of candidate .gitmodules. Add repositories
|
||
# here only with an explicit review of the checkout credential's scope.
|
||
REPOSITORIES = {
|
||
"modules/core": "Agent-System-Core",
|
||
"modules/backend": "Agent-System-Backend",
|
||
"modules/frontend": "Agent-System-Frontend",
|
||
"modules/ui": "Agent-System-UI",
|
||
"modules/documents": "Agent-System-Documents",
|
||
"deps/agent-system-core": "Agent-System-Core",
|
||
"deps/development-guidelines": "Development-Guidelines",
|
||
"deps/agent-native": "Agent-Native",
|
||
"deps/agent-squad": "Agent-Squad",
|
||
"deps/agent-intent": "Agent-Intent",
|
||
"deps/agent-command": "Agent-Command",
|
||
"deps/error-code-system": "Error-Code-System",
|
||
"deps/logging-system": "Logging-System",
|
||
}
|
||
root_token = os.environ.pop("TOKEN", "")
|
||
component_token = os.environ.pop("SUBMODULE_TOKEN", "")
|
||
if not root_token or not component_token:
|
||
sys.exit("Checkout requires TOKEN and SUBMODULE_TOKEN")
|
||
if os.environ.get("GITHUB_REPOSITORY") != "weichen/Agent-System-Dev":
|
||
sys.exit("Checkout repository is outside the reviewed allowlist")
|
||
sha = os.environ.get("GITHUB_SHA", "")
|
||
if not re.fullmatch(r"[0-9a-fA-F]{40}", sha):
|
||
sys.exit("Checkout requires a fixed commit SHA")
|
||
|
||
# Discard inherited Git tracing/config/SSH hooks. Auth lives only in one
|
||
# Git process tree, in an exact URL-scoped header, never in a URL or file.
|
||
clean_env = {k: v for k, v in os.environ.items() if not k.startswith("GIT_")}
|
||
clean_env.update(GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_SYSTEM="/dev/null",
|
||
GIT_CONFIG_GLOBAL="/dev/null", GIT_TERMINAL_PROMPT="0")
|
||
config = {
|
||
"core.hooksPath": "/dev/null", "core.fsmonitor": "false",
|
||
"init.templateDir": "", "credential.helper": "",
|
||
"fetch.recurseSubmodules": "false", "submodule.recurse": "false",
|
||
"http.followRedirects": "false", "protocol.allow": "never",
|
||
"protocol." + urlsplit(SERVER).scheme + ".allow": "always",
|
||
}
|
||
|
||
|
||
def git(repo, *args, extra=None):
|
||
values = config | (extra or {})
|
||
env = clean_env | {"GIT_CONFIG_COUNT": str(len(values))}
|
||
for index, (key, value) in enumerate(values.items()):
|
||
env[f"GIT_CONFIG_KEY_{index}"] = key
|
||
env[f"GIT_CONFIG_VALUE_{index}"] = value
|
||
return subprocess.run(["git", "-C", str(repo), *args], env=env,
|
||
check=True, stdout=subprocess.PIPE).stdout
|
||
|
||
|
||
def materialize(repo, depth=0):
|
||
if depth > 16:
|
||
raise ValueError("Submodule depth exceeds the reviewed checkout limit")
|
||
links = {}
|
||
for entry in git(repo, "ls-tree", "-rz", "HEAD").split(b"\0"):
|
||
if not entry:
|
||
continue
|
||
metadata, path = entry.split(b"\t", 1)
|
||
mode, kind, commit = metadata.decode().split()
|
||
if mode == "160000":
|
||
links[path.decode()] = commit
|
||
if not links:
|
||
return
|
||
raw = git(repo, "config", "--no-includes", "--blob", "HEAD:.gitmodules", "--null", "--list")
|
||
entries = {}
|
||
for item in raw.split(b"\0"):
|
||
if not item:
|
||
continue
|
||
key, value = item.decode().split("\n", 1)
|
||
match = re.fullmatch(r"submodule\.(.+)\.(path|url)", key)
|
||
if not match:
|
||
raise ValueError("Unsupported candidate .gitmodules setting")
|
||
name, field = match.groups()
|
||
if field in entries.setdefault(name, {}):
|
||
raise ValueError("Duplicate candidate .gitmodules setting")
|
||
entries[name][field] = value
|
||
if set(entries) != set(links):
|
||
raise ValueError("Submodule names must match the fixed gitlink paths")
|
||
# Validate the entire level before any authenticated component fetch.
|
||
for name, fields in entries.items():
|
||
repository = REPOSITORIES.get(name)
|
||
if not repository or fields.get("path") != name:
|
||
raise ValueError("Submodule path is outside the reviewed allowlist")
|
||
trusted_url = f"{SERVER}/weichen/{repository}.git"
|
||
if fields.get("url") not in {SSH + repository + ".git", trusted_url}:
|
||
raise ValueError("Submodule URL is outside the reviewed allowlist")
|
||
for name in entries:
|
||
url = f"{SERVER}/weichen/{REPOSITORIES[name]}.git"
|
||
# Persist only the reviewed public URL so Git recognizes the
|
||
# initialized submodule after process-scoped config is gone.
|
||
git(repo, "config", f"submodule.{name}.url", url)
|
||
git(repo, "submodule", "update", "--init", "--checkout", "--", name,
|
||
extra={f"submodule.{name}.url": url,
|
||
f"submodule.{name}.update": "checkout",
|
||
f"http.{url}.extraheader": "AUTHORIZATION: token " + component_token})
|
||
child = repo / name
|
||
if git(child, "rev-parse", "HEAD").decode().strip() != links[name]:
|
||
raise ValueError("Materialized submodule does not match the fixed SHA")
|
||
materialize(child, depth + 1)
|
||
|
||
|
||
try:
|
||
root = Path.cwd()
|
||
# An ephemeral clean checkout is required; do not reuse prior Git state.
|
||
if (root / ".git").exists():
|
||
raise ValueError("Checkout requires a fresh workspace")
|
||
git(root, "init", "-q")
|
||
url = SERVER + "/weichen/Agent-System-Dev.git"
|
||
git(root, "remote", "add", "origin", url)
|
||
git(root, "fetch", "--depth=1", "origin", sha,
|
||
extra={f"http.{url}.extraheader": "AUTHORIZATION: token " + root_token})
|
||
git(root, "checkout", "-q", "--detach", "FETCH_HEAD")
|
||
materialize(root)
|
||
except (ValueError, OSError, subprocess.CalledProcessError) as error:
|
||
sys.exit(f"Checkout refused: {error}")
|
||
print("Checkout complete; credentials were not persisted in Git configuration")
|
||
PY_CHECKOUT
|
||
unset TOKEN SUBMODULE_TOKEN
|
||
- name: Install Go 1.27.1
|
||
run: |
|
||
set -euo pipefail
|
||
curl -fsSL -o "$RUNNER_TEMP/go.tgz" https://mirrors.aliyun.com/golang/go1.27.1.linux-amd64.tar.gz
|
||
tar -C /usr/local -xzf "$RUNNER_TEMP/go.tgz"
|
||
ln -sfn /usr/local/go/bin/go /usr/local/bin/go
|
||
ln -sfn /usr/local/go/bin/gofmt /usr/local/bin/gofmt
|
||
- name: Install Node 22
|
||
run: |
|
||
set -euo pipefail
|
||
curl -fsSL -o "$RUNNER_TEMP/node.tar.xz" https://npmmirror.com/mirrors/node/v22.23.3/node-v22.23.3-linux-x64.tar.xz
|
||
tar -C /usr/local --strip-components=1 -xJf "$RUNNER_TEMP/node.tar.xz"
|
||
- name: Install house-style Python packages
|
||
run: python3 -m pip install --break-system-packages -i https://pypi.tuna.tsinghua.edu.cn/simple python-docx python-pptx openpyxl
|
||
- name: make test (boundaries, pins, libs, core, backend, frontend, ui, agent-command)
|
||
run: make test
|
||
- name: Build the single binary
|
||
run: make build
|
||
|
||
# 协作存储方案 U2：真实 PostgreSQL + pgvector + Redis 门禁。
|
||
# 缺服务时失败而非跳过（make test-backend-pg 硬校验环境变量）；
|
||
# 组件版本随 job 输出固定，供验收记录对齐（方案 KTD12/U2）。
|
||
test-pg:
|
||
runs-on: ubuntu-latest
|
||
env:
|
||
GOPROXY: https://goproxy.cn,direct
|
||
steps:
|
||
- name: Checkout
|
||
env:
|
||
TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||
SUBMODULE_TOKEN: ${{ secrets.SUBMODULE_TOKEN }}
|
||
run: |
|
||
set -euo pipefail
|
||
# Credential handling stays inline: no file from the candidate is executed.
|
||
# Workflow provenance and runner isolation still require platform controls.
|
||
python3 - <<'PY_CHECKOUT'
|
||
import os
|
||
from pathlib import Path
|
||
import re
|
||
import subprocess
|
||
import sys
|
||
from urllib.parse import urlsplit
|
||
|
||
# 服务端基址取运行器注入的 GITHUB_SERVER_URL（平台控制、候选 PR 不可影响）。
|
||
# 2026-10-05 修复（run 948/949 实测）：注入值为内网直连形态
|
||
# 'http://172.17.0.1:3000'（docker 网关 → 宿主 Gitea），与公网拼写不同且为
|
||
# http——此前硬编码常量严格比对会拒绝合法检出。校验收宽为「http(s) + 非空
|
||
# netloc」；凭据仍只附着于「该基址 + 下方仓库白名单」构造的 URL。
|
||
SERVER = os.environ.get("GITHUB_SERVER_URL", "").rstrip("/")
|
||
parsed_server = urlsplit(SERVER)
|
||
if parsed_server.scheme not in {"http", "https"} or not parsed_server.netloc:
|
||
sys.exit("Checkout server must be the platform-injected http(s) URL")
|
||
SSH = "ssh://git@git-ssh.u1680832.nyat.app:31411/weichen/"
|
||
# Reviewed allowlist, independent of candidate .gitmodules. Add repositories
|
||
# here only with an explicit review of the checkout credential's scope.
|
||
REPOSITORIES = {
|
||
"modules/core": "Agent-System-Core",
|
||
"modules/backend": "Agent-System-Backend",
|
||
"modules/frontend": "Agent-System-Frontend",
|
||
"modules/ui": "Agent-System-UI",
|
||
"modules/documents": "Agent-System-Documents",
|
||
"deps/agent-system-core": "Agent-System-Core",
|
||
"deps/development-guidelines": "Development-Guidelines",
|
||
"deps/agent-native": "Agent-Native",
|
||
"deps/agent-squad": "Agent-Squad",
|
||
"deps/agent-intent": "Agent-Intent",
|
||
"deps/agent-command": "Agent-Command",
|
||
"deps/error-code-system": "Error-Code-System",
|
||
"deps/logging-system": "Logging-System",
|
||
}
|
||
root_token = os.environ.pop("TOKEN", "")
|
||
component_token = os.environ.pop("SUBMODULE_TOKEN", "")
|
||
if not root_token or not component_token:
|
||
sys.exit("Checkout requires TOKEN and SUBMODULE_TOKEN")
|
||
if os.environ.get("GITHUB_REPOSITORY") != "weichen/Agent-System-Dev":
|
||
sys.exit("Checkout repository is outside the reviewed allowlist")
|
||
sha = os.environ.get("GITHUB_SHA", "")
|
||
if not re.fullmatch(r"[0-9a-fA-F]{40}", sha):
|
||
sys.exit("Checkout requires a fixed commit SHA")
|
||
|
||
# Discard inherited Git tracing/config/SSH hooks. Auth lives only in one
|
||
# Git process tree, in an exact URL-scoped header, never in a URL or file.
|
||
clean_env = {k: v for k, v in os.environ.items() if not k.startswith("GIT_")}
|
||
clean_env.update(GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_SYSTEM="/dev/null",
|
||
GIT_CONFIG_GLOBAL="/dev/null", GIT_TERMINAL_PROMPT="0")
|
||
config = {
|
||
"core.hooksPath": "/dev/null", "core.fsmonitor": "false",
|
||
"init.templateDir": "", "credential.helper": "",
|
||
"fetch.recurseSubmodules": "false", "submodule.recurse": "false",
|
||
"http.followRedirects": "false", "protocol.allow": "never",
|
||
"protocol." + urlsplit(SERVER).scheme + ".allow": "always",
|
||
}
|
||
|
||
|
||
def git(repo, *args, extra=None):
|
||
values = config | (extra or {})
|
||
env = clean_env | {"GIT_CONFIG_COUNT": str(len(values))}
|
||
for index, (key, value) in enumerate(values.items()):
|
||
env[f"GIT_CONFIG_KEY_{index}"] = key
|
||
env[f"GIT_CONFIG_VALUE_{index}"] = value
|
||
return subprocess.run(["git", "-C", str(repo), *args], env=env,
|
||
check=True, stdout=subprocess.PIPE).stdout
|
||
|
||
|
||
def materialize(repo, depth=0):
|
||
if depth > 16:
|
||
raise ValueError("Submodule depth exceeds the reviewed checkout limit")
|
||
links = {}
|
||
for entry in git(repo, "ls-tree", "-rz", "HEAD").split(b"\0"):
|
||
if not entry:
|
||
continue
|
||
metadata, path = entry.split(b"\t", 1)
|
||
mode, kind, commit = metadata.decode().split()
|
||
if mode == "160000":
|
||
links[path.decode()] = commit
|
||
if not links:
|
||
return
|
||
raw = git(repo, "config", "--no-includes", "--blob", "HEAD:.gitmodules", "--null", "--list")
|
||
entries = {}
|
||
for item in raw.split(b"\0"):
|
||
if not item:
|
||
continue
|
||
key, value = item.decode().split("\n", 1)
|
||
match = re.fullmatch(r"submodule\.(.+)\.(path|url)", key)
|
||
if not match:
|
||
raise ValueError("Unsupported candidate .gitmodules setting")
|
||
name, field = match.groups()
|
||
if field in entries.setdefault(name, {}):
|
||
raise ValueError("Duplicate candidate .gitmodules setting")
|
||
entries[name][field] = value
|
||
if set(entries) != set(links):
|
||
raise ValueError("Submodule names must match the fixed gitlink paths")
|
||
# Validate the entire level before any authenticated component fetch.
|
||
for name, fields in entries.items():
|
||
repository = REPOSITORIES.get(name)
|
||
if not repository or fields.get("path") != name:
|
||
raise ValueError("Submodule path is outside the reviewed allowlist")
|
||
trusted_url = f"{SERVER}/weichen/{repository}.git"
|
||
if fields.get("url") not in {SSH + repository + ".git", trusted_url}:
|
||
raise ValueError("Submodule URL is outside the reviewed allowlist")
|
||
for name in entries:
|
||
url = f"{SERVER}/weichen/{REPOSITORIES[name]}.git"
|
||
# Persist only the reviewed public URL so Git recognizes the
|
||
# initialized submodule after process-scoped config is gone.
|
||
git(repo, "config", f"submodule.{name}.url", url)
|
||
git(repo, "submodule", "update", "--init", "--checkout", "--", name,
|
||
extra={f"submodule.{name}.url": url,
|
||
f"submodule.{name}.update": "checkout",
|
||
f"http.{url}.extraheader": "AUTHORIZATION: token " + component_token})
|
||
child = repo / name
|
||
if git(child, "rev-parse", "HEAD").decode().strip() != links[name]:
|
||
raise ValueError("Materialized submodule does not match the fixed SHA")
|
||
materialize(child, depth + 1)
|
||
|
||
|
||
try:
|
||
root = Path.cwd()
|
||
# An ephemeral clean checkout is required; do not reuse prior Git state.
|
||
if (root / ".git").exists():
|
||
raise ValueError("Checkout requires a fresh workspace")
|
||
git(root, "init", "-q")
|
||
url = SERVER + "/weichen/Agent-System-Dev.git"
|
||
git(root, "remote", "add", "origin", url)
|
||
git(root, "fetch", "--depth=1", "origin", sha,
|
||
extra={f"http.{url}.extraheader": "AUTHORIZATION: token " + root_token})
|
||
git(root, "checkout", "-q", "--detach", "FETCH_HEAD")
|
||
materialize(root)
|
||
except (ValueError, OSError, subprocess.CalledProcessError) as error:
|
||
sys.exit(f"Checkout refused: {error}")
|
||
print("Checkout complete; credentials were not persisted in Git configuration")
|
||
PY_CHECKOUT
|
||
unset TOKEN SUBMODULE_TOKEN
|
||
- name: Install Go 1.27.1
|
||
run: |
|
||
set -euo pipefail
|
||
curl -fsSL -o "$RUNNER_TEMP/go.tgz" https://mirrors.aliyun.com/golang/go1.27.1.linux-amd64.tar.gz
|
||
tar -C /usr/local -xzf "$RUNNER_TEMP/go.tgz"
|
||
ln -sfn /usr/local/go/bin/go /usr/local/bin/go
|
||
ln -sfn /usr/local/go/bin/gofmt /usr/local/bin/gofmt
|
||
- name: Install PostgreSQL, pgvector and Redis
|
||
run: |
|
||
set -euo pipefail
|
||
SUDO=""
|
||
if command -v sudo >/dev/null 2>&1 && [ [ "$(id -u)" != 0 ]; then SUDO=sudo; fi
|
||
$SUDO apt-get update -y
|
||
$SUDO apt-get install -y postgresql postgresql-contrib postgresql-16-pgvector redis-server
|
||
# 固定组件/扩展版本输出（随日志归档，作为验收对齐依据）。
|
||
psql --version
|
||
$SUDO -u postgres psql -tAc "SELECT extversion FROM pg_extension WHERE extname='pgvector'" || true
|
||
redis-server --version
|
||
- name: Start PostgreSQL and Redis
|
||
run: |
|
||
set -euo pipefail
|
||
SUDO=""
|
||
if command -v sudo >/dev/null 2>&1 && [ "$(id -u)" != 0 ]; then SUDO=sudo; fi
|
||
$SUDO service postgresql start || $SUDO pg_ctlcluster 16 main start
|
||
$SUDO -u postgres psql -c "CREATE USER ci_agent PASSWORD 'ci_job_local_only' CREATEDB"
|
||
$SUDO -u postgres psql -c "CREATE DATABASE agent_test OWNER ci_agent"
|
||
# 预装 pg_trgm 到 public：与测试辅助的幂等预装一致，锁定的组件契约。
|
||
$SUDO -u postgres psql -d agent_test -c "CREATE EXTENSION IF NOT EXISTS pg_trgm"
|
||
redis-server --daemonize yes --port 6399 --save '' --appendonly no
|
||
redis-cli -p 6399 ping
|
||
- name: Backend tests against real PostgreSQL and Redis (required, no silent skip)
|
||
env:
|
||
AGENT_PG_TEST_DSN: postgres://ci_agent:ci_job_local_only@127.0.0.1:5432/agent_test?sslmode=disable
|
||
AGENT_REDIS_TEST_ADDR: 127.0.0.1:6399
|
||
run: make test-backend-pg
|