weichen
eb4796cf61
ci / vet-and-test (pull_request) Successful in 46s
Why: plan U5 remainder (KTD5/KTD9 + exit criteria) - commands must commit atomically with business writes, artifacts need a shared object store for multi-instance, and the two-real-process acceptance matrix must run in CI. Evidence: full suite green on SQLite and real PostgreSQL 18.6 (35 packages each tier) - outbox tx test (rollback removes the command, commit makes both visible, idempotent re-enqueue), S3 store tests (SigV4-signed round trip, 404 sentinel, idempotent delete, presigned GET, env fallback), real-process acceptance (testbinary self-spawn of two serve processes with per-process identity and command consumers against one schema: same-conversation contention, cross-instance cancel command claimed exactly once, stale-owner settle fenced with run unchanged). Impact: EnqueueCommandInTx for in-transaction command outbox and AppendEvents now notifies in the same transaction (KTD6 front half); blobstore.S3 (SigV4, presign, prefix isolation) injected from BLOBSTORE_S3_* env when configured, local fallback otherwise; realproc_test provides the process-level multi-instance acceptance gate (PG-env gated, runs in CI test-pg job). Refs: proposals/2026-10-11-collaboration-storage-and-distribution.md U5 KTD5/KTD9, section 5 exit criteria; R4/R7/R10.
243 lines
7.6 KiB
Go
243 lines
7.6 KiB
Go
package blobstore
|
||
|
||
import (
|
||
"bytes"
|
||
"crypto/hmac"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"net/http"
|
||
"net/url"
|
||
"os"
|
||
"strings"
|
||
"time"
|
||
)
|
||
|
||
// S3 是 S3 兼容对象存储实现（协作存储 U5/KTD9：多实例共享工件）。
|
||
// 认证：AWS Signature V4（静态 AccessKey/SecretKey；凭据只经环境或受保护
|
||
// 配置注入，引用形态为对象键，不含凭据）。
|
||
//
|
||
// 语义（KTD9）：
|
||
// - Put：先完整写对象再由上层提交引用（失败孤儿可清理）；
|
||
// - Get：按对象键读取；
|
||
// - Delete：立即删除（标记/延迟 GC 由桶生命周期策略承担）。
|
||
type S3 struct {
|
||
Base string // 端点（含 bucket 路径，如 https://s3.example.com/bucket）
|
||
Region string
|
||
Access string
|
||
Secret string
|
||
Prefix string // 对象键前缀（租户/应用隔离）
|
||
client *http.Client
|
||
nowFunc func() time.Time
|
||
}
|
||
|
||
// NewS3 从显式参数构造；nil client 用默认。
|
||
func NewS3(base, region, access, secret, prefix string) (*S3, error) {
|
||
if base == "" || access == "" || secret == "" {
|
||
return nil, errors.New("blobstore: s3 requires base/access/secret")
|
||
}
|
||
if !strings.HasSuffix(base, "/") {
|
||
base += "/"
|
||
}
|
||
return &S3{Base: base, Region: region, Access: access, Secret: secret, Prefix: prefix,
|
||
client: &http.Client{Timeout: 30 * time.Second}, nowFunc: time.Now}, nil
|
||
}
|
||
|
||
// NewS3FromEnv 按环境构造：BLOBSTORE_S3_ENDPOINT/REGION/ACCESS_KEY/SECRET_KEY/PREFIX。
|
||
// 未配置端点返回 (nil, nil)（调用方回落本地档）。
|
||
func NewS3FromEnv() (*S3, error) {
|
||
if os.Getenv("BLOBSTORE_S3_ENDPOINT") == "" {
|
||
return nil, nil
|
||
}
|
||
return NewS3(os.Getenv("BLOBSTORE_S3_ENDPOINT"),
|
||
envOr("BLOBSTORE_S3_REGION", "us-east-1"),
|
||
os.Getenv("BLOBSTORE_S3_ACCESS_KEY"),
|
||
os.Getenv("BLOBSTORE_S3_SECRET_KEY"),
|
||
os.Getenv("BLOBSTORE_S3_PREFIX"))
|
||
}
|
||
|
||
func envOr(k, d string) string {
|
||
if v := os.Getenv(k); v != "" {
|
||
return v
|
||
}
|
||
return d
|
||
}
|
||
|
||
func (s *S3) key(id string) (string, error) {
|
||
if !validID(id) {
|
||
return "", fmt.Errorf("blobstore: invalid object id %q", id)
|
||
}
|
||
return s.Prefix + id, nil
|
||
}
|
||
|
||
// Put 上传对象；ref=对象键。
|
||
func (s *S3) Put(id string, content []byte) (string, error) {
|
||
key, err := s.key(id)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
resp, err := s.do(http.MethodPut, key, content, "")
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
defer resp.Body.Close()
|
||
if resp.StatusCode/100 != 2 {
|
||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 256))
|
||
return "", fmt.Errorf("blobstore: s3 put %s: %s %s", key, resp.Status, body)
|
||
}
|
||
return key, nil
|
||
}
|
||
|
||
// Get 下载对象。
|
||
func (s *S3) Get(ref string) ([]byte, error) {
|
||
resp, err := s.do(http.MethodGet, ref, nil, "")
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer resp.Body.Close()
|
||
if resp.StatusCode == http.StatusNotFound {
|
||
return nil, ErrObjectNotFound
|
||
}
|
||
if resp.StatusCode/100 != 2 {
|
||
return nil, fmt.Errorf("blobstore: s3 get %s: %s", ref, resp.Status)
|
||
}
|
||
return io.ReadAll(resp.Body)
|
||
}
|
||
|
||
// ErrObjectNotFound 对象不存在（与本地档 ErrNotExist 对齐的哨兵）。
|
||
var ErrObjectNotFound = errors.New("blobstore: object not found")
|
||
|
||
// Delete 删除对象（幂等：404 视为成功）。
|
||
func (s *S3) Delete(ref string) error {
|
||
resp, err := s.do(http.MethodDelete, ref, nil, "")
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer resp.Body.Close()
|
||
if resp.StatusCode/100 == 2 || resp.StatusCode == http.StatusNotFound {
|
||
return nil
|
||
}
|
||
return fmt.Errorf("blobstore: s3 delete %s: %s", ref, resp.Status)
|
||
}
|
||
|
||
// do 签名并执行请求（AWS SigV4，service=s3）。
|
||
func (s *S3) do(method, key string, body []byte, contentType string) (*http.Response, error) {
|
||
u, err := url.Parse(s.Base + key)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
now := s.nowFunc().UTC()
|
||
amzDate := now.Format("20060102T150405Z")
|
||
dateStamp := now.Format("20060102")
|
||
payloadHash := sha256.Sum256(body)
|
||
payloadHex := hex.EncodeToString(payloadHash[:])
|
||
|
||
req, err := http.NewRequest(method, u.String(), bytes.NewReader(body))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
req.Header.Set("x-amz-date", amzDate)
|
||
req.Header.Set("x-amz-content-sha256", payloadHex)
|
||
if contentType != "" {
|
||
req.Header.Set("Content-Type", contentType)
|
||
}
|
||
// 规范请求与签名（SigV4 单 chunk）。
|
||
canonicalURI := "/" + strings.TrimPrefix(u.Path, "/")
|
||
canonicalQuery := u.RawQuery
|
||
var ch bytes.Buffer
|
||
ch.WriteString(method + "\n" + canonicalURI + "\n" + canonicalQuery + "\n")
|
||
ch.WriteString("host:" + u.Host + "\n")
|
||
ch.WriteString("x-amz-content-sha256:" + payloadHex + "\n")
|
||
ch.WriteString("x-amz-date:" + amzDate + "\n")
|
||
ch.WriteString("\nhost;x-amz-content-sha256;x-amz-date\n" + payloadHex)
|
||
cr := ch.Bytes()
|
||
|
||
scope := dateStamp + "/" + s.Region + "/s3/aws4_request"
|
||
stringToSign := "AWS4-HMAC-SHA256\n" + amzDate + "\n" + scope + "\n" + hex.EncodeToString(sha256sum(cr))
|
||
kDate := hmacSHA256([]byte("AWS4"+s.Secret), dateStamp)
|
||
kRegion := hmacSHA256(kDate, s.Region)
|
||
kService := hmacSHA256(kRegion, "s3")
|
||
kSigning := hmacSHA256(kService, "aws4_request")
|
||
sig := hex.EncodeToString(hmacSHA256(kSigning, stringToSign))
|
||
req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential="+s.Access+"/"+scope+
|
||
", SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature="+sig)
|
||
return s.client.Do(req)
|
||
}
|
||
|
||
func sha256sum(b []byte) []byte {
|
||
h := sha256.Sum256(b)
|
||
return h[:]
|
||
}
|
||
|
||
func hmacSHA256(key []byte, data string) []byte {
|
||
m := hmac.New(sha256.New, key)
|
||
m.Write([]byte(data))
|
||
return m.Sum(nil)
|
||
}
|
||
|
||
// PresignGet 生成限时 GET 预签名 URL（下载入口；有效期由调用方指定）。
|
||
// 用于把大对象下载直接指向对象存储，应用实例不做数据中继。
|
||
func (s *S3) PresignGet(ref string, ttl time.Duration) (string, error) {
|
||
u, err := url.Parse(s.Base + ref)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
now := s.nowFunc().UTC()
|
||
amzDate := now.Format("20060102T150405Z")
|
||
dateStamp := now.Format("20060102")
|
||
expiry := int64(ttl / time.Second)
|
||
payloadHex := hex.EncodeToString(sha256sum(nil))
|
||
q := u.Query()
|
||
q.Set("X-Amz-Algorithm", "AWS4-HMAC-SHA256")
|
||
q.Set("X-Amz-Credential", s.Access+"/"+dateStamp+"/"+s.Region+"/s3/aws4_request")
|
||
q.Set("X-Amz-Date", amzDate)
|
||
q.Set("X-Amz-Expires", fmt.Sprint(expiry))
|
||
q.Set("X-Amz-SignedHeaders", "host")
|
||
u.RawQuery = q.Encode()
|
||
|
||
canonicalQuery := canonicalQueryEncode(u)
|
||
var ch bytes.Buffer
|
||
ch.WriteString("GET\n/" + strings.TrimPrefix(u.Path, "/") + "\n" + canonicalQuery + "\n")
|
||
ch.WriteString("host:" + u.Host + "\n")
|
||
ch.WriteString("x-amz-content-sha256:" + payloadHex + "\n")
|
||
ch.WriteString("x-amz-date:" + amzDate + "\n")
|
||
ch.WriteString("\nhost;x-amz-content-sha256;x-amz-date\n" + payloadHex)
|
||
scope := dateStamp + "/" + s.Region + "/s3/aws4_request"
|
||
stringToSign := "AWS4-HMAC-SHA256\n" + amzDate + "\n" + scope + "\n" + hex.EncodeToString(sha256sum(ch.Bytes()))
|
||
kDate := hmacSHA256([]byte("AWS4"+s.Secret), dateStamp)
|
||
kRegion := hmacSHA256(kDate, s.Region)
|
||
kService := hmacSHA256(kRegion, "s3")
|
||
kSigning := hmacSHA256(kService, "aws4_request")
|
||
sig := hex.EncodeToString(hmacSHA256(kSigning, stringToSign))
|
||
return u.String() + "&X-Amz-Signature=" + sig, nil
|
||
}
|
||
|
||
// canonicalQueryEncode 按 AWS 规则重排查询串（键排序 + URL 编码）。
|
||
func canonicalQueryEncode(u *url.URL) string {
|
||
q := u.Query()
|
||
var keys []string
|
||
for k := range q {
|
||
keys = append(keys, k)
|
||
}
|
||
// 排序（简单实现足够：键集固定且互不冲突）
|
||
for i := 0; i < len(keys); i++ {
|
||
for j := i + 1; j < len(keys); j++ {
|
||
if keys[j] < keys[i] {
|
||
keys[i], keys[j] = keys[j], keys[i]
|
||
}
|
||
}
|
||
}
|
||
var sb strings.Builder
|
||
for i, k := range keys {
|
||
if i > 0 {
|
||
sb.WriteString("&")
|
||
}
|
||
sb.WriteString(url.QueryEscape(k) + "=" + url.QueryEscape(q[k][0]))
|
||
}
|
||
return sb.String()
|
||
}
|
||
|
||
var _ Store = (*S3)(nil)
|