feat(artifactvault): extract audited FAV with union-deny reads

Goose 00036 is a renumbered copy of the vault tables. Degraded read stays off by default and only runs after authorization; tombstones and unpublished artifacts fail closed.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
weichen
2026-09-14 15:59:36 +00:00
co-authored by Cursor
parent f27059b453
commit c7e3e99303
20 changed files with 2472 additions and 2 deletions
+33 -1
View File
@@ -60,6 +60,7 @@ imports:
- docgov
- docquery
- docrevision
- events
app/uow: []
docquery:
- docgov
@@ -129,6 +130,7 @@ imports:
- quote
- stream
- template
- workspace
storage: []
stream: []
taskrun:
@@ -148,7 +150,13 @@ imports:
- storage
- upload
- workflow
workspace: []
workspace:
- app/uow
events:
- app/uow
artifactvault:
- app/uow
- fsutil
table_owners:
document_records:
@@ -167,6 +175,26 @@ table_owners:
- docrevision
document_change_events:
- docrevision
event_envelopes:
- events
event_deliveries:
- events
event_dead_letters:
- events
final_artifacts:
- artifactvault
artifact_payload_staging:
- artifactvault
artifact_blob_references:
- artifactvault
artifact_publication_attempts:
- artifactvault
artifact_deletion_ledger:
- artifactvault
artifact_quarantine:
- artifactvault
artifact_backfill_exclusions:
- artifactvault
path_owners:
output.md:
@@ -175,6 +203,7 @@ path_owners:
data/dev:
- config
- workspace
- artifactvault
workspaces/:
- squad
- workspace
@@ -188,6 +217,7 @@ event_producers:
document.agent_version_committed:
- docrevision
- app/docflow
- events
document.downstream_impact_detected:
- docrevision
document.change_report_degraded:
@@ -203,6 +233,8 @@ unwrap_owners:
- docrevision
- taskrun
- freedoc
- events
- artifactvault
exceptions:
- kind: sql_write
@@ -0,0 +1,94 @@
-- +goose Up
-- Renumbered from feature/final-artifact-vault 00034 after audit. Do not merge that file as 00034.
CREATE TABLE IF NOT EXISTS final_artifacts (
artifact_uuid TEXT PRIMARY KEY,
idempotency_key TEXT NOT NULL UNIQUE,
artifact_type TEXT NOT NULL,
logical_resource_id TEXT NOT NULL,
environment TEXT NOT NULL,
project_id BIGINT,
project_external_key TEXT NOT NULL,
source_event_id TEXT NOT NULL,
source_event_kind TEXT NOT NULL DEFAULT '',
source_snapshot_id BIGINT,
source_message_id BIGINT,
source_run_id TEXT NOT NULL DEFAULT '',
payload_sha256 CHAR(64) NOT NULL,
manifest_sha256 CHAR(64),
vault_storage_identity TEXT,
terminal_state TEXT NOT NULL DEFAULT '',
policy_version TEXT NOT NULL DEFAULT '',
publication_status TEXT NOT NULL,
retention_class TEXT NOT NULL DEFAULT 'governance',
requested_at TIMESTAMPTZ NOT NULL DEFAULT now(),
verified_at TIMESTAMPTZ,
tombstoned_at TIMESTAMPTZ,
CONSTRAINT chk_fav_type CHECK (artifact_type IN ('agent_commit','sealed_version','approved_document','freeform_publish','assistant_reply')),
CONSTRAINT chk_fav_status CHECK (publication_status IN ('requested','publishing','verified','retryable_failed','tombstoned','purged','quarantine'))
);
CREATE TABLE IF NOT EXISTS artifact_payload_staging (
staging_uuid TEXT PRIMARY KEY,
project_id BIGINT,
source_event_id TEXT NOT NULL,
payload_sha256 CHAR(64) NOT NULL,
encrypted_payload BYTEA NOT NULL,
classification TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
releasable_at TIMESTAMPTZ,
UNIQUE (source_event_id, payload_sha256)
);
CREATE TABLE IF NOT EXISTS artifact_blob_references (
artifact_uuid TEXT NOT NULL REFERENCES final_artifacts(artifact_uuid),
project_scope TEXT NOT NULL,
payload_sha256 CHAR(64) NOT NULL,
envelope_key_ref TEXT NOT NULL DEFAULT '',
reference_status TEXT NOT NULL DEFAULT 'active',
retention_class TEXT NOT NULL DEFAULT 'governance',
legal_hold BOOLEAN NOT NULL DEFAULT FALSE,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (artifact_uuid, payload_sha256)
);
CREATE TABLE IF NOT EXISTS artifact_publication_attempts (
artifact_uuid TEXT NOT NULL,
idempotency_key TEXT NOT NULL,
attempt_no INT NOT NULL,
error_code TEXT NOT NULL DEFAULT '',
receipt TEXT NOT NULL DEFAULT '',
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (artifact_uuid, attempt_no)
);
CREATE TABLE IF NOT EXISTS artifact_deletion_ledger (
artifact_uuid TEXT NOT NULL,
reason TEXT NOT NULL,
authority TEXT NOT NULL DEFAULT '',
effective_at TIMESTAMPTZ NOT NULL,
applied_at TIMESTAMPTZ,
audit_ref TEXT NOT NULL DEFAULT ''
);
CREATE TABLE IF NOT EXISTS artifact_quarantine (
logical_resource_id TEXT NOT NULL,
reason TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS artifact_backfill_exclusions (
logical_resource_id TEXT PRIMARY KEY,
reason TEXT NOT NULL,
approved_by TEXT NOT NULL,
expires_at TIMESTAMPTZ NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
-- +goose Down
DROP TABLE IF EXISTS artifact_backfill_exclusions;
DROP TABLE IF EXISTS artifact_quarantine;
DROP TABLE IF EXISTS artifact_deletion_ledger;
DROP TABLE IF EXISTS artifact_publication_attempts;
DROP TABLE IF EXISTS artifact_blob_references;
DROP TABLE IF EXISTS artifact_payload_staging;
DROP TABLE IF EXISTS final_artifacts;
+85
View File
@@ -0,0 +1,85 @@
package docflow_test
import (
"context"
"errors"
"os"
"strings"
"testing"
"time"
"chat-agent-system/internal/app/uow"
"chat-agent-system/internal/artifactvault"
"chat-agent-system/internal/db"
"chat-agent-system/internal/events"
"github.com/jackc/pgx/v5/pgxpool"
)
func init() {
if _, err := os.Stat("deploy/postgres/goose"); err == nil {
return
}
for _, cand := range []string{"../../..", "../../../.."} {
if fi, err := os.Stat(cand + "/deploy/postgres/goose"); err == nil && fi.IsDir() {
_ = os.Chdir(cand)
return
}
}
}
func TestOutboxAndStagingSameTxRollback(t *testing.T) {
if testing.Short() {
t.Skip("same-tx outbox/staging is not part of -short")
}
dsn := db.SkipIfNoTestDatabaseURL(t)
if strings.Contains(dsn, "5433") || strings.Contains(dsn, "agent_system_dev") ||
(strings.Contains(dsn, "5434") && strings.Contains(dsn, "agent_system_test")) ||
strings.Contains(dsn, "5435") {
t.Fatal("TEST_DATABASE_URL points at a shared env port; refuse agent_system_dev/test/main")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, dsn)
if err != nil {
t.Fatal(err)
}
t.Cleanup(pool.Close)
if err := db.RunMigrationsOnceForTest(ctx, pool, dsn); err != nil {
t.Fatalf("migrate: %v", err)
}
body := []byte(`{"doc_id":"doc_m2_tx","version_no":1}`)
env := events.Envelope{
EventID: "document:doc_m2_tx:v1", EventName: "document.agent_version_committed",
SchemaVersion: events.EnvelopeSchema, Environment: "dev", ProjectID: 9,
AggregateType: "document", AggregateID: "doc_m2_tx", AggregateVersion: 1,
ActorRef: "user:1", IdempotencyKey: "commit:doc_m2_tx:1", PayloadDigest: events.Digest(body),
Producer: "docflow", OccurredAt: time.Now().UTC(), Payload: body,
}
boom := errors.New("boom after staging")
unit := uow.NewPG(pool)
err = unit.Within(ctx, func(scope uow.TxScope) error {
ctx = uow.WithScope(ctx, scope)
if err := (events.PGSink{}).Append(ctx, env, []string{"projection", "fav"}); err != nil {
return err
}
ing := &artifactvault.Ingest{Env: "dev"}
if err := ing.StageAgentCommit(ctx, 9, "doc_m2_tx", env.EventID, 4, body); err != nil {
return err
}
return boom
})
if !errors.Is(err, boom) {
t.Fatalf("want boom, got %v", err)
}
var envelopes, artifacts int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM event_envelopes WHERE event_id=$1`, env.EventID).Scan(&envelopes); err != nil {
t.Fatal(err)
}
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM final_artifacts WHERE source_event_id=$1`, env.EventID).Scan(&artifacts); err != nil {
t.Fatal(err)
}
if envelopes != 0 || artifacts != 0 {
t.Fatalf("rollback leaked envelopes=%d artifacts=%d", envelopes, artifacts)
}
}
+30
View File
@@ -0,0 +1,30 @@
package artifactvault
import "fmt"
func CoveragePASS(c Coverage) bool {
if c.Required <= 0 {
return false
}
return c.Verified == c.Required && c.Quarantine == 0
}
func CanPurge(legalHold bool, activeRefs int) error {
if legalHold {
return fmt.Errorf("artifactvault: legal hold")
}
if activeRefs > 0 {
return fmt.Errorf("artifactvault: remaining references")
}
return nil
}
func PreflightDestroy(c Coverage, backupOK, corpusOK, vaultReady bool) error {
if !vaultReady || !CoveragePASS(c) {
return fmt.Errorf("artifactvault: coverage gate failed required=%d verified=%d quarantine=%d", c.Required, c.Verified, c.Quarantine)
}
if !backupOK || !corpusOK {
return fmt.Errorf("artifactvault: complementary backup/corpus not PASS")
}
return nil
}
+53
View File
@@ -0,0 +1,53 @@
package artifactvault
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
)
// ProjectKey derives a per-project envelope key. The master secret never enters manifests.
func ProjectKey(master []byte, env, projectExternalKey string) (key []byte, ref string, err error) {
if len(master) < 16 {
return nil, "", fmt.Errorf("artifactvault: vault master key too short")
}
sum := sha256.Sum256(append(append(master, []byte(env)...), []byte(projectExternalKey)...))
ref = "proj:" + env + ":" + projectExternalKey + ":" + hex.EncodeToString(sum[:8])
return sum[:], ref, nil
}
func Encrypt(key, plaintext []byte) ([]byte, error) {
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return nil, err
}
nonce := make([]byte, gcm.NonceSize())
if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
return nil, err
}
return gcm.Seal(nonce, nonce, plaintext, nil), nil
}
func Decrypt(key, envelope []byte) ([]byte, error) {
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return nil, err
}
ns := gcm.NonceSize()
if len(envelope) < ns {
return nil, fmt.Errorf("artifactvault: short envelope")
}
return gcm.Open(nil, envelope[:ns], envelope[ns:], nil)
}
+108
View File
@@ -0,0 +1,108 @@
package artifactvault
import (
"errors"
"strings"
)
var (
ErrNotTerminal = errors.New("artifactvault: facts are not a terminal state")
ErrExplicitBypass = errors.New("artifactvault: explicit publish/pin cannot bypass terminal allowlist")
ErrSensitiveRejected = errors.New("artifactvault: sensitive or unclassified payload rejected")
ErrUnknownArtifactType = errors.New("artifactvault: unknown artifact type rejected")
ErrSnapshotInconsistent = errors.New("artifactvault: snapshot/event inconsistent")
ErrUnionDeny = errors.New("artifactvault: union-deny")
)
// EvaluateEligibility is the only domain gate. Unclassified types default to reject.
func EvaluateEligibility(f Facts) error {
if err := rejectSensitive(f.Classification, f.Payload); err != nil {
return err
}
switch f.ArtifactType {
case TypeAgentCommit:
if f.CandidateStatus != "committed" || f.OutputSnapshotID <= 0 {
return ErrNotTerminal
}
if f.ChangeKind != "" && (f.ChangeKind != "agent_commit" || f.ChangeToSnapshotID != f.OutputSnapshotID) {
return ErrSnapshotInconsistent
}
return nil
case TypeSealedVersion:
if f.VersionStatus != "sealed" || f.SealedAt == nil || f.HeadSnapshotID <= 0 {
return ErrNotTerminal
}
return nil
case TypeApprovedDocument:
if f.DocState != "approved" || f.ApprovedAt == nil || f.ReviewSnapshotID <= 0 {
return ErrNotTerminal
}
return nil
case TypeFreeformPublish:
if !f.ExplicitPublish || !f.HasImmutableSnapshot {
if f.ExplicitPublish && !f.HasImmutableSnapshot {
return ErrExplicitBypass
}
return ErrNotTerminal
}
return nil
case TypeAssistantReply:
if f.ChatTurnStatus != "completed" || f.CompletedAt == nil || f.AssistantMessageID <= 0 {
if f.ExplicitPin {
return ErrExplicitBypass
}
return ErrNotTerminal
}
switch f.ExportPolicy {
case "full":
return nil
case "copy_only":
if f.ExplicitPin && f.PinAfterCompleted {
return nil
}
if f.ExplicitPin {
return ErrExplicitBypass
}
return ErrNotTerminal
default:
return ErrNotTerminal
}
default:
return ErrUnknownArtifactType
}
}
func rejectSensitive(class string, payload []byte) error {
if class == "" {
class = ClassifyPayload(payload)
}
switch class {
case ClassificationSecret, ClassificationRestricted, ClassificationFailed:
return ErrSensitiveRejected
case ClassificationOK:
return nil
default:
return ErrSensitiveRejected
}
}
// ClassifyPayload is a fail-closed server-side gate. It never logs the body.
func ClassifyPayload(payload []byte) string {
text := strings.ToLower(string(payload))
if text == "" {
return ClassificationFailed
}
needles := []string{
"-----begin ", "private_key", "api_key=", "secret=", "authorization: bearer",
"password=", "access_token=", "aws_secret",
}
for _, n := range needles {
if strings.Contains(text, n) {
return ClassificationSecret
}
}
if strings.Contains(text, "restricted:") || strings.Contains(text, "classification_failed") {
return ClassificationRestricted
}
return ClassificationOK
}
+115
View File
@@ -0,0 +1,115 @@
package artifactvault
import (
"testing"
"time"
)
func TestEvaluateEligibility_Allowlist(t *testing.T) {
ok := Facts{
ArtifactType: TypeAgentCommit, CandidateStatus: "committed", OutputSnapshotID: 9,
ChangeKind: "agent_commit", ChangeToSnapshotID: 9, Classification: ClassificationOK, Payload: []byte("doc"),
}
if err := EvaluateEligibility(ok); err != nil {
t.Fatalf("committed should pass: %v", err)
}
for _, status := range []string{"ready", "needs_confirmation", "rejected", "stale", "quality_failed"} {
bad := ok
bad.CandidateStatus = status
if err := EvaluateEligibility(bad); err == nil {
t.Fatalf("status %s must reject", status)
}
}
}
func TestEvaluateEligibility_ApprovedRequiresReviewSnapshot(t *testing.T) {
now := time.Now().UTC()
f := Facts{
ArtifactType: TypeApprovedDocument, DocState: "approved", ApprovedAt: &now,
Classification: ClassificationOK, Payload: []byte("approved"),
}
if err := EvaluateEligibility(f); err == nil {
t.Fatal("approved without review_snapshot_id must reject")
}
f.ReviewSnapshotID = 3
if err := EvaluateEligibility(f); err != nil {
t.Fatalf("approved with snapshot should pass: %v", err)
}
}
func TestEvaluateEligibility_TaskWithoutCompletedTurn(t *testing.T) {
f := Facts{
ArtifactType: TypeAssistantReply, ChatTurnStatus: "", AssistantMessageID: 8,
ExportPolicy: "full", Classification: ClassificationOK, Payload: []byte("hello world from assistant"),
}
if err := EvaluateEligibility(f); err == nil {
t.Fatal("missing completed turn must reject")
}
if err := EvaluateEligibility(Facts{
ArtifactType: TypeAssistantReply, ExplicitPin: true, ChatTurnStatus: "pending",
Classification: ClassificationOK, Payload: []byte("hello world from assistant"),
}); err != ErrExplicitBypass {
t.Fatalf("pin before completed: %v", err)
}
}
func TestEvaluateEligibility_SensitiveRejectedBeforeStaging(t *testing.T) {
now := time.Now().UTC()
f := Facts{
ArtifactType: TypeAssistantReply, ChatTurnStatus: "completed", CompletedAt: &now,
AssistantMessageID: 1, ExportPolicy: "full", Payload: []byte("api_key=sk-secret"),
}
if err := EvaluateEligibility(f); err != ErrSensitiveRejected {
t.Fatalf("secret payload: %v", err)
}
}
func TestEvaluateEligibility_CopyOnlyRequiresExplicitPinAfterCompleted(t *testing.T) {
now := time.Now().UTC()
base := Facts{
ArtifactType: TypeAssistantReply, ChatTurnStatus: "completed", CompletedAt: &now,
AssistantMessageID: 9, ExportPolicy: "copy_only", Classification: ClassificationOK,
Payload: []byte("ok, noted."),
}
if err := EvaluateEligibility(base); err == nil {
t.Fatal("copy_only without pin must reject")
}
base.ExplicitPin = true
if err := EvaluateEligibility(base); err != ErrExplicitBypass {
t.Fatalf("pin without PinAfterCompleted: %v", err)
}
base.PinAfterCompleted = true
if err := EvaluateEligibility(base); err != nil {
t.Fatalf("copy_only + pin after completed should pass: %v", err)
}
}
func TestEvaluateEligibility_UnknownTypeRejected(t *testing.T) {
if err := EvaluateEligibility(Facts{ArtifactType: "workspace_scan", Classification: ClassificationOK, Payload: []byte("x")}); err != ErrUnknownArtifactType {
t.Fatalf("got %v", err)
}
}
func TestEvaluateEligibility_IntermediateAdmissionZero(t *testing.T) {
for _, typ := range []string{"prompt", "tool_trace", "scratch", "workspace_scan"} {
if err := EvaluateEligibility(Facts{ArtifactType: typ, Classification: ClassificationOK, Payload: []byte("x")}); err != ErrUnknownArtifactType {
t.Fatalf("%s: %v", typ, err)
}
}
}
func TestIdempotencyKeyStable(t *testing.T) {
req := PublishRequest{
ArtifactType: TypeAgentCommit, Environment: "dev", ProjectExternalKey: "p1",
LogicalResourceID: "doc_a", SourceEventKind: "agent_commit", SourceEventID: "ev1",
}
a := IdempotencyKey(req, "abc")
b := IdempotencyKey(req, "abc")
if a != b || a == "" {
t.Fatalf("key unstable %q %q", a, b)
}
req.SourceEventID = "ev2"
if IdempotencyKey(req, "abc") == a {
t.Fatal("new event must new key")
}
}
+217
View File
@@ -0,0 +1,217 @@
package artifactvault
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"chat-agent-system/internal/fsutil"
)
type FilesystemStore struct {
Root string
StorageIdentity string
MasterKey []byte
}
func NewFilesystemStore(root, identity string, master []byte) (*FilesystemStore, error) {
if strings.TrimSpace(root) == "" || strings.Contains(root, "data/dev") && !strings.Contains(root, "vault") {
if strings.HasSuffix(filepath.Clean(root), filepath.Join("data", "dev")) {
return nil, fmt.Errorf("artifactvault: refuse vault root %s (must not be runtime data/dev)", root)
}
}
if identity == "" {
return nil, fmt.Errorf("artifactvault: storage identity required")
}
if err := os.MkdirAll(root, 0o700); err != nil {
return nil, err
}
sent := filepath.Join(root, ".vault-storage-identity.json")
if b, err := os.ReadFile(sent); err == nil {
var got map[string]string
_ = json.Unmarshal(b, &got)
if got["uuid"] != "" && got["uuid"] != identity {
return nil, fmt.Errorf("artifactvault: storage identity mismatch")
}
} else {
_ = fsutil.WriteFileAtomic(sent, []byte(`{"uuid":"`+identity+`"}`), 0o444)
}
return &FilesystemStore{Root: root, StorageIdentity: identity, MasterKey: master}, nil
}
func (s *FilesystemStore) Identity() string { return s.StorageIdentity }
func (s *FilesystemStore) blobPath(env, projectKey, sha string) string {
return filepath.Join(s.Root, "scopes", env, projectKey, "blobs", "sha256", sha[:2], sha[2:4], sha)
}
func (s *FilesystemStore) PutBlob(env, projectKey, sha string, plain []byte) (keyRef string, err error) {
if SHA256Hex(plain) != sha {
return "", fmt.Errorf("artifactvault: blob sha mismatch")
}
key, ref, err := ProjectKey(s.MasterKey, env, projectKey)
if err != nil {
return "", err
}
envBytes, err := Encrypt(key, plain)
if err != nil {
return "", err
}
path := s.blobPath(env, projectKey, sha)
if _, err := os.Stat(path); err == nil {
return ref, nil
}
if err := writeExclusiveOK(path, envBytes, 0o600); err != nil {
return "", err
}
return ref, nil
}
func (s *FilesystemStore) GetBlob(env, projectKey, sha string) ([]byte, error) {
key, _, err := ProjectKey(s.MasterKey, env, projectKey)
if err != nil {
return nil, err
}
raw, err := os.ReadFile(s.blobPath(env, projectKey, sha))
if err != nil {
return nil, err
}
plain, err := Decrypt(key, raw)
if err != nil {
return nil, err
}
if SHA256Hex(plain) != sha {
return nil, fmt.Errorf("artifactvault: decrypted sha mismatch")
}
return plain, nil
}
func writeExclusiveOK(path string, raw []byte, perm os.FileMode) error {
if err := fsutil.WriteFileAtomicExclusive(path, raw, perm); err != nil {
if strings.Contains(err.Error(), "refuse overwrite") {
return nil
}
return err
}
return nil
}
func (s *FilesystemStore) PutManifest(artifactUUID, manifestSHA string, raw []byte) error {
path := filepath.Join(s.Root, "manifests", artifactUUID, manifestSHA+".json")
return writeExclusiveOK(path, raw, 0o644)
}
func (s *FilesystemStore) PutCommit(artifactUUID, manifestSHA string, raw []byte) error {
path := filepath.Join(s.Root, "commits", artifactUUID, manifestSHA+".json")
return writeExclusiveOK(path, raw, 0o644)
}
func (s *FilesystemStore) ListCommitted() ([]string, error) {
dir := filepath.Join(s.Root, "commits")
ents, err := os.ReadDir(dir)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
out := make([]string, 0, len(ents))
for _, e := range ents {
if e.IsDir() {
out = append(out, e.Name())
}
}
return out, nil
}
func (s *FilesystemStore) LoadCommittedArtifacts() ([]Artifact, map[string][]byte, error) {
uuids, err := s.ListCommitted()
if err != nil {
return nil, nil, err
}
payloads := make(map[string][]byte)
arts := make([]Artifact, 0, len(uuids))
for _, uuid := range uuids {
if s.HasTombstone(uuid) {
arts = append(arts, Artifact{UUID: uuid, PublicationStatus: StatusTombstoned})
continue
}
art, body, err := s.loadCommitted(uuid)
if err != nil {
return nil, nil, err
}
arts = append(arts, art)
payloads[uuid] = body
}
return arts, payloads, nil
}
func (s *FilesystemStore) loadCommitted(uuid string) (Artifact, []byte, error) {
dir := filepath.Join(s.Root, "commits", uuid)
ents, err := os.ReadDir(dir)
if err != nil || len(ents) == 0 {
return Artifact{}, nil, fmt.Errorf("artifactvault: unpublished")
}
raw, err := os.ReadFile(filepath.Join(dir, ents[0].Name()))
if err != nil {
return Artifact{}, nil, err
}
var marker CommitMarker
if err := json.Unmarshal(raw, &marker); err != nil {
return Artifact{}, nil, err
}
manRaw, err := os.ReadFile(filepath.Join(s.Root, "manifests", uuid, marker.ManifestSHA256+".json"))
if err != nil {
return Artifact{}, nil, err
}
var man Manifest
if err := json.Unmarshal(manRaw, &man); err != nil {
return Artifact{}, nil, err
}
if len(man.PayloadMembers) == 0 {
return Artifact{}, nil, fmt.Errorf("artifactvault: manifest has no payload members")
}
body, err := s.GetBlob(man.Environment, man.ProjectExternalKey, man.PayloadMembers[0].SHA256)
if err != nil {
return Artifact{}, nil, err
}
var snap, msg int64
if man.SourceSnapshotID != nil {
snap = *man.SourceSnapshotID
}
if man.SourceMessageID != nil {
msg = *man.SourceMessageID
}
return Artifact{
UUID: uuid, ArtifactType: man.ArtifactType, LogicalResourceID: man.LogicalResourceID,
Environment: man.Environment, ProjectID: man.ProjectID, ProjectExternalKey: man.ProjectExternalKey,
SourceEventID: man.SourceEventID, SourceEventKind: man.SourceEventKind,
SourceSnapshotID: snap, SourceMessageID: msg, SourceRunID: man.SourceRunID,
PayloadSHA256: man.PayloadMembers[0].SHA256, ManifestSHA256: man.ManifestSHA256,
VaultStorageIdentity: man.VaultStorageIdentity, TerminalState: man.TerminalEvidence.State,
PublicationStatus: StatusVerified, RetentionClass: man.RetentionClass,
}, body, nil
}
func (s *FilesystemStore) HasCommit(artifactUUID, manifestSHA string) bool {
_, err := os.Stat(filepath.Join(s.Root, "commits", artifactUUID, manifestSHA+".json"))
return err == nil
}
func (s *FilesystemStore) PutTombstone(artifactUUID, eventUUID string, raw []byte) error {
path := filepath.Join(s.Root, "tombstones", artifactUUID, eventUUID+".json")
return fsutil.WriteFileAtomicExclusive(path, raw, 0o644)
}
func (s *FilesystemStore) HasTombstone(artifactUUID string) bool {
dir := filepath.Join(s.Root, "tombstones", artifactUUID)
ents, err := os.ReadDir(dir)
return err == nil && len(ents) > 0
}
func (s *FilesystemStore) Ready() (identityOK, connected, lagOK bool) {
_, err := os.Stat(s.Root)
return s.StorageIdentity != "", err == nil, err == nil
}
+202
View File
@@ -0,0 +1,202 @@
package artifactvault
import (
"context"
"fmt"
"strconv"
"time"
"chat-agent-system/internal/app/uow"
"github.com/jackc/pgx/v5"
)
// Ingest is the single write path from domain terminals into staging/outbox.
type Ingest struct {
Env string
Reg Registry
}
func ProjectKeyFromID(projectID int64) string {
if projectID <= 0 {
return "unscoped"
}
return strconv.FormatInt(projectID, 10)
}
func (i *Ingest) Request(ctx context.Context, facts Facts, req PublishRequest) error {
if i == nil || i.Reg == nil {
return nil
}
if req.Environment == "" {
req.Environment = i.Env
}
if req.ProjectExternalKey == "" {
req.ProjectExternalKey = ProjectKeyFromID(req.ProjectID)
}
if req.PolicyVersion == "" {
req.PolicyVersion = PolicyVersion
}
if req.Classification == "" {
req.Classification = ClassifyPayload(req.Payload)
}
facts.Payload = req.Payload
facts.Classification = req.Classification
_, _, err := i.Reg.Request(ctx, facts, req)
return err
}
func ApprovedFacts(docState string, approvedAt *time.Time, snapshotID int64) Facts {
return Facts{
ArtifactType: TypeApprovedDocument,
DocState: docState,
ApprovedAt: approvedAt,
ReviewSnapshotID: snapshotID,
}
}
func AgentCommitFacts(outputSnapshotID int64) Facts {
return Facts{
ArtifactType: TypeAgentCommit,
CandidateStatus: "committed",
OutputSnapshotID: outputSnapshotID,
ChangeKind: "agent_commit",
ChangeToSnapshotID: outputSnapshotID,
}
}
func SealedFacts(sealedAt *time.Time, headSnapshotID int64) Facts {
return Facts{
ArtifactType: TypeSealedVersion,
VersionStatus: "sealed",
SealedAt: sealedAt,
HeadSnapshotID: headSnapshotID,
}
}
func AssistantFacts(turnStatus string, completedAt *time.Time, msgID int64, policy string) Facts {
return Facts{
ArtifactType: TypeAssistantReply,
ChatTurnStatus: turnStatus,
CompletedAt: completedAt,
AssistantMessageID: msgID,
ExportPolicy: policy,
}
}
func FreeformFacts(explicit bool, hasSnapshot bool) Facts {
return Facts{
ArtifactType: TypeFreeformPublish,
ExplicitPublish: explicit,
HasImmutableSnapshot: hasSnapshot,
}
}
func (i *Ingest) AgentCommit(ctx context.Context, projectID int64, docID, eventID string, snapshotID int64, payload []byte) error {
return i.Request(ctx, AgentCommitFacts(snapshotID), PublishRequest{
ArtifactType: TypeAgentCommit, Environment: i.Env, ProjectID: projectID,
LogicalResourceID: docID, SourceEventKind: "agent_commit", SourceEventID: eventID,
SourceSnapshotID: snapshotID, Payload: payload, TerminalState: "committed",
})
}
func (i *Ingest) SealedVersion(ctx context.Context, projectID int64, docID, eventID string, snapshotID int64, sealedAt *time.Time, payload []byte) error {
return i.Request(ctx, SealedFacts(sealedAt, snapshotID), PublishRequest{
ArtifactType: TypeSealedVersion, Environment: i.Env, ProjectID: projectID,
LogicalResourceID: docID + ":sealed", SourceEventKind: "version_sealed", SourceEventID: eventID,
SourceSnapshotID: snapshotID, Payload: payload, TerminalState: "sealed",
})
}
func (i *Ingest) Approved(ctx context.Context, projectID int64, docID string, snapshotID int64, approvedAt *time.Time, payload []byte) error {
return i.Request(ctx, ApprovedFacts("approved", approvedAt, snapshotID), PublishRequest{
ArtifactType: TypeApprovedDocument, Environment: i.Env, ProjectID: projectID,
LogicalResourceID: docID, SourceEventKind: "approved", SourceEventID: fmt.Sprintf("approved:%s:%d", docID, snapshotID),
SourceSnapshotID: snapshotID, Payload: payload, TerminalState: "approved",
})
}
func (i *Ingest) FreeformPublish(ctx context.Context, projectID int64, docID, eventID string, snapshotID int64, payload []byte) error {
return i.Request(ctx, FreeformFacts(true, snapshotID > 0 || len(payload) > 0), PublishRequest{
ArtifactType: TypeFreeformPublish, Environment: i.Env, ProjectID: projectID,
LogicalResourceID: docID, SourceEventKind: "freeform_publish", SourceEventID: eventID,
SourceSnapshotID: snapshotID, Payload: payload, TerminalState: "published",
})
}
func (i *Ingest) requestTX(ctx context.Context, tx pgx.Tx, facts Facts, req PublishRequest) error {
if i == nil {
return nil
}
if req.Environment == "" {
req.Environment = i.Env
}
if req.ProjectExternalKey == "" {
req.ProjectExternalKey = ProjectKeyFromID(req.ProjectID)
}
if req.PolicyVersion == "" {
req.PolicyVersion = PolicyVersion
}
if req.Classification == "" {
req.Classification = ClassifyPayload(req.Payload)
}
facts.Payload = req.Payload
facts.Classification = req.Classification
_, _, err := RequestWithTx(ctx, tx, facts, req)
return err
}
func (i *Ingest) StageAgentCommit(ctx context.Context, projectID int64, docID, eventID string, snapshotID int64, payload []byte) error {
scope, ok := uow.ScopeFrom(ctx)
if !ok {
return fmt.Errorf("artifactvault: staging requires a unit of work")
}
tx, err := uow.AsPG(scope)
if err != nil {
return err
}
return i.AgentCommitTX(ctx, tx, projectID, docID, eventID, snapshotID, payload)
}
func (i *Ingest) AgentCommitTX(ctx context.Context, tx pgx.Tx, projectID int64, docID, eventID string, snapshotID int64, payload []byte) error {
return i.requestTX(ctx, tx, AgentCommitFacts(snapshotID), PublishRequest{
ArtifactType: TypeAgentCommit, Environment: i.Env, ProjectID: projectID,
LogicalResourceID: docID, SourceEventKind: "agent_commit", SourceEventID: eventID,
SourceSnapshotID: snapshotID, Payload: payload, TerminalState: "committed",
})
}
func (i *Ingest) SealedVersionTX(ctx context.Context, tx pgx.Tx, projectID int64, docID, eventID string, snapshotID int64, sealedAt *time.Time, payload []byte) error {
return i.requestTX(ctx, tx, SealedFacts(sealedAt, snapshotID), PublishRequest{
ArtifactType: TypeSealedVersion, Environment: i.Env, ProjectID: projectID,
LogicalResourceID: docID + ":sealed", SourceEventKind: "version_sealed", SourceEventID: eventID,
SourceSnapshotID: snapshotID, Payload: payload, TerminalState: "sealed",
})
}
func (i *Ingest) ApprovedTX(ctx context.Context, tx pgx.Tx, projectID int64, docID string, snapshotID int64, approvedAt *time.Time, payload []byte) error {
return i.requestTX(ctx, tx, ApprovedFacts("approved", approvedAt, snapshotID), PublishRequest{
ArtifactType: TypeApprovedDocument, Environment: i.Env, ProjectID: projectID,
LogicalResourceID: docID, SourceEventKind: "approved", SourceEventID: fmt.Sprintf("approved:%s:%d", docID, snapshotID),
SourceSnapshotID: snapshotID, Payload: payload, TerminalState: "approved",
})
}
func (i *Ingest) AssistantReply(ctx context.Context, projectID int64, logical, eventID string, msgID int64, runID, policy string, completedAt *time.Time, payload []byte) error {
return i.Request(ctx, AssistantFacts("completed", completedAt, msgID, policy), PublishRequest{
ArtifactType: TypeAssistantReply, Environment: i.Env, ProjectID: projectID,
LogicalResourceID: logical, SourceEventKind: "assistant_completed", SourceEventID: eventID,
SourceMessageID: msgID, SourceRunID: runID, Payload: payload, TerminalState: "completed",
})
}
func (i *Ingest) AssistantReplyPin(ctx context.Context, projectID int64, logical, eventID string, msgID int64, runID, policy string, completedAt *time.Time, payload []byte) error {
facts := AssistantFacts("completed", completedAt, msgID, policy)
facts.ExplicitPin = true
facts.PinAfterCompleted = true
return i.Request(ctx, facts, PublishRequest{
ArtifactType: TypeAssistantReply, Environment: i.Env, ProjectID: projectID,
LogicalResourceID: logical, SourceEventKind: "assistant_pin", SourceEventID: eventID,
SourceMessageID: msgID, SourceRunID: runID, Payload: payload, TerminalState: "completed",
})
}
+35
View File
@@ -0,0 +1,35 @@
package artifactvault
import (
"context"
"testing"
"time"
)
func TestIngestIdempotentAndRejectsNonTerminal(t *testing.T) {
ctx := context.Background()
ing := &Ingest{Env: "dev", Reg: NewMemoryRegistry()}
now := time.Now().UTC()
payload := []byte("# committed body")
if err := ing.AgentCommit(ctx, 9, "doc_a", "ev-1", 4, payload); err != nil {
t.Fatal(err)
}
if err := ing.AgentCommit(ctx, 9, "doc_a", "ev-1", 4, payload); err != nil {
t.Fatal(err)
}
if ing.Reg.Coverage(ctx).Required != 1 {
t.Fatalf("idempotent ingest required=%d", ing.Reg.Coverage(ctx).Required)
}
if err := ing.AssistantReply(ctx, 9, "msg:1", "s:t", 1, "run", "none", &now, []byte("hello world from assistant")); err == nil {
t.Fatal("none policy must reject")
}
}
func TestCoverageGateFailClosed(t *testing.T) {
if CoveragePASS(Coverage{}) {
t.Fatal("empty coverage must fail")
}
if err := PreflightDestroy(Coverage{Required: 2, Verified: 1}, true, true, true); err == nil {
t.Fatal("incomplete coverage must fail destroy")
}
}
+100
View File
@@ -0,0 +1,100 @@
package artifactvault
import (
"bytes"
"encoding/json"
"fmt"
"strings"
"time"
)
func BuildManifest(art Artifact, payloadSHA string, size int, keyRef string) Manifest {
var snap, msg *int64
if art.SourceSnapshotID > 0 {
v := art.SourceSnapshotID
snap = &v
}
if art.SourceMessageID > 0 {
v := art.SourceMessageID
msg = &v
}
m := Manifest{
ArtifactSchemaVersion: SchemaMajorMinor,
ArtifactUUID: art.UUID,
ArtifactType: art.ArtifactType,
LogicalResourceID: art.LogicalResourceID,
Environment: art.Environment,
ProjectID: art.ProjectID,
ProjectExternalKey: art.ProjectExternalKey,
SourceEventID: art.SourceEventID,
SourceEventKind: art.SourceEventKind,
SourceSnapshotID: snap,
SourceMessageID: msg,
SourceRunID: art.SourceRunID,
PayloadMembers: []PayloadMember{{
Name: "payload", MediaType: "text/markdown", ByteSize: size, SHA256: payloadSHA, Required: true,
}},
TerminalEvidence: TerminalEvidence{
State: art.TerminalState, DecidedAt: time.Now().UTC().Format(time.RFC3339), PolicyVersion: nz(art.PolicyVersion, PolicyVersion),
},
RetentionClass: art.RetentionClass,
EncryptionKeyRef: keyRef,
VaultStorageIdentity: art.VaultStorageIdentity,
PublisherVersion: PublisherVersion,
PublishedAt: time.Now().UTC().Format(time.RFC3339),
}
raw, _ := json.Marshal(canonicalManifest(m))
m.ManifestSHA256 = SHA256Hex(raw)
return m
}
func canonicalManifest(m Manifest) Manifest {
cp := m
cp.ManifestSHA256 = ""
return cp
}
func ValidateManifest(m Manifest) error {
if !strings.HasPrefix(m.ArtifactSchemaVersion, "1.") {
return fmt.Errorf("artifactvault: unsupported manifest major %q", m.ArtifactSchemaVersion)
}
if m.ArtifactUUID == "" || m.LogicalResourceID == "" || m.ProjectExternalKey == "" || m.VaultStorageIdentity == "" {
return fmt.Errorf("artifactvault: manifest missing required identity fields")
}
if len(m.PayloadMembers) == 0 {
return fmt.Errorf("artifactvault: manifest has no payload members")
}
for _, mem := range m.PayloadMembers {
if len(mem.SHA256) != 64 {
return fmt.Errorf("artifactvault: invalid member sha")
}
}
got := SHA256Hex(mustJSON(canonicalManifest(m)))
if m.ManifestSHA256 != "" && m.ManifestSHA256 != got {
return fmt.Errorf("artifactvault: manifest sha mismatch")
}
return nil
}
func ValidateCommitMarker(c CommitMarker) error {
if c.ArtifactUUID == "" || len(c.ManifestSHA256) != 64 || len(c.MemberSHAs) == 0 || c.VaultStorageIdentity == "" || c.CommittedAt == "" || c.PublisherVersion == "" {
return fmt.Errorf("artifactvault: incomplete commit marker")
}
return nil
}
func mustJSON(v any) []byte {
b, err := json.Marshal(v)
if err != nil {
return nil
}
return b
}
func EncodeJSON(v any) []byte {
var buf bytes.Buffer
enc := json.NewEncoder(&buf)
enc.SetEscapeHTML(false)
_ = enc.Encode(v)
return bytes.TrimSpace(buf.Bytes())
}
+103
View File
@@ -0,0 +1,103 @@
package artifactvault
import (
"context"
"encoding/json"
"fmt"
"time"
)
type Publisher struct {
Reg Registry
Store *FilesystemStore
}
func (p *Publisher) Drain(ctx context.Context) error {
arts, err := p.Reg.LoadRequested(ctx)
if err != nil {
return err
}
var first error
for _, a := range arts {
if err := p.PublishOne(ctx, a); err != nil && first == nil {
first = err
}
}
return first
}
func (p *Publisher) PublishOne(ctx context.Context, art Artifact) error {
if p.Store == nil {
return fmt.Errorf("artifactvault: store required")
}
if p.Reg.IsDenied(ctx, art.UUID) {
return fmt.Errorf("artifactvault: artifact denied")
}
_ = p.Reg.MarkPublishing(ctx, art.UUID)
st, err := p.Reg.LoadStaging(ctx, art.SourceEventID)
if err != nil {
_ = p.Reg.MarkFailed(ctx, art.UUID, "staging_missing")
return err
}
if SHA256Hex(st.Payload) != art.PayloadSHA256 || SHA256Hex(st.Payload) != st.PayloadSHA256 {
_ = p.Reg.MarkFailed(ctx, art.UUID, "sha_mismatch")
return fmt.Errorf("artifactvault: staging sha mismatch")
}
if ClassifyPayload(st.Payload) != ClassificationOK {
_ = p.Reg.MarkFailed(ctx, art.UUID, "sensitive")
return ErrSensitiveRejected
}
art.VaultStorageIdentity = p.Store.Identity()
keyRef, err := p.Store.PutBlob(art.Environment, art.ProjectExternalKey, art.PayloadSHA256, st.Payload)
if err != nil {
_ = p.Reg.MarkFailed(ctx, art.UUID, "blob")
return err
}
man := BuildManifest(art, art.PayloadSHA256, len(st.Payload), keyRef)
if err := ValidateManifest(man); err != nil {
_ = p.Reg.MarkFailed(ctx, art.UUID, "manifest")
return err
}
manRaw := EncodeJSON(man)
if err := p.Store.PutManifest(art.UUID, man.ManifestSHA256, manRaw); err != nil {
_ = p.Reg.MarkFailed(ctx, art.UUID, "manifest_write")
return err
}
marker := CommitMarker{
ArtifactUUID: art.UUID, ManifestSHA256: man.ManifestSHA256,
MemberSHAs: []string{art.PayloadSHA256}, VaultStorageIdentity: p.Store.Identity(),
CommittedAt: time.Now().UTC().Format(time.RFC3339), PublisherVersion: PublisherVersion,
}
if err := ValidateCommitMarker(marker); err != nil {
return err
}
if err := p.Store.PutCommit(art.UUID, man.ManifestSHA256, EncodeJSON(marker)); err != nil {
_ = p.Reg.MarkFailed(ctx, art.UUID, "commit")
return err
}
_ = p.Reg.RecordReference(ctx, art.UUID, art.ProjectExternalKey, art.PayloadSHA256, keyRef)
return p.Reg.MarkVerified(ctx, art.UUID, man.ManifestSHA256)
}
func (p *Publisher) DegradedPayload(ctx context.Context, env, logical, projectKey string) ([]byte, string, error) {
art, err := p.Reg.GetByLogical(ctx, env, logical)
if err != nil {
return nil, "", err
}
if p.Reg.IsDenied(ctx, art.UUID) || p.Store.HasTombstone(art.UUID) {
return nil, "", ErrUnionDeny
}
if art.ManifestSHA256 == "" || !p.Store.HasCommit(art.UUID, art.ManifestSHA256) {
return nil, "", fmt.Errorf("artifactvault: unpublished")
}
b, err := p.Store.GetBlob(env, projectKey, art.PayloadSHA256)
if err != nil {
return nil, "", err
}
return b, art.PayloadSHA256, nil
}
func MustJSON(v any) string {
b, _ := json.Marshal(v)
return string(b)
}
+267
View File
@@ -0,0 +1,267 @@
package artifactvault
import (
"context"
"errors"
"os"
"path/filepath"
"testing"
"time"
)
func testStore(t *testing.T) *FilesystemStore {
t.Helper()
dir := t.TempDir()
st, err := NewFilesystemStore(dir, "vault-id-test", []byte("0123456789abcdef0123456789abcdef"))
if err != nil {
t.Fatal(err)
}
return st
}
func TestPublisherIdempotentAndCrashSafe(t *testing.T) {
ctx := context.Background()
reg := NewMemoryRegistry()
store := testStore(t)
pub := &Publisher{Reg: reg, Store: store}
facts := Facts{
ArtifactType: TypeAgentCommit, CandidateStatus: "committed", OutputSnapshotID: 4,
ChangeKind: "agent_commit", ChangeToSnapshotID: 4, Classification: ClassificationOK,
Payload: []byte("# final\n"),
}
req := PublishRequest{
ArtifactType: TypeAgentCommit, Environment: "dev", ProjectExternalKey: "proj-a",
LogicalResourceID: "doc_a", SourceEventKind: "agent_commit", SourceEventID: "ev-1",
Payload: []byte("# final\n"), TerminalState: "committed",
}
art1, _, err := reg.Request(ctx, facts, req)
if err != nil {
t.Fatal(err)
}
art2, _, err := reg.Request(ctx, facts, req)
if err != nil || art1.UUID != art2.UUID {
t.Fatalf("idempotent request: %v %s %s", err, art1.UUID, art2.UUID)
}
if err := pub.Drain(ctx); err != nil {
t.Fatal(err)
}
if err := pub.Drain(ctx); err != nil {
t.Fatal(err)
}
cov := reg.Coverage(ctx)
if cov.Verified != 1 {
t.Fatalf("verified=%d", cov.Verified)
}
b, sha, err := pub.DegradedPayload(ctx, "dev", "doc_a", "proj-a")
if err != nil || string(b) != "# final\n" || sha == "" {
t.Fatalf("degraded: %v %q", err, b)
}
}
func TestPublisherRejectsBadSHAAndCrossProjectKeys(t *testing.T) {
ctx := context.Background()
reg := NewMemoryRegistry()
store := testStore(t)
pub := &Publisher{Reg: reg, Store: store}
facts := Facts{
ArtifactType: TypeSealedVersion, VersionStatus: "sealed", SealedAt: ptrTime(),
HeadSnapshotID: 2, Classification: ClassificationOK, Payload: []byte("same-bytes"),
}
aReq := PublishRequest{ArtifactType: TypeSealedVersion, Environment: "dev", ProjectExternalKey: "p1", LogicalResourceID: "d1", SourceEventKind: "seal", SourceEventID: "s1", Payload: []byte("same-bytes"), TerminalState: "sealed"}
bReq := aReq
bReq.ProjectExternalKey = "p2"
bReq.LogicalResourceID = "d2"
bReq.SourceEventID = "s2"
if _, _, err := reg.Request(ctx, facts, aReq); err != nil {
t.Fatal(err)
}
if _, _, err := reg.Request(ctx, facts, bReq); err != nil {
t.Fatal(err)
}
if err := pub.Drain(ctx); err != nil {
t.Fatal(err)
}
if reg.ActiveRefs(ctx, "p1", SHA256Hex([]byte("same-bytes"))) != 1 {
t.Fatal("p1 refs")
}
if reg.ActiveRefs(ctx, "p2", SHA256Hex([]byte("same-bytes"))) != 1 {
t.Fatal("p2 refs")
}
_ = reg.Tombstone(ctx, "unused", "test", "test")
}
func TestFilesystemRefusesRuntimeDataDev(t *testing.T) {
root := filepath.Join(t.TempDir(), "data", "dev")
_ = os.MkdirAll(root, 0o755)
if _, err := NewFilesystemStore(root, "id", []byte("0123456789abcdef0123456789abcdef")); err == nil {
t.Fatal("must refuse data/dev")
}
}
func TestTombstoneUnionDeny(t *testing.T) {
ctx := context.Background()
reg := NewMemoryRegistry()
store := testStore(t)
pub := &Publisher{Reg: reg, Store: store}
now := time.Now().UTC()
facts := Facts{ArtifactType: TypeApprovedDocument, DocState: "approved", ApprovedAt: &now, ReviewSnapshotID: 8, Classification: ClassificationOK, Payload: []byte("ok body")}
req := PublishRequest{ArtifactType: TypeApprovedDocument, Environment: "dev", ProjectExternalKey: "p", LogicalResourceID: "doc", SourceEventKind: "approved", SourceEventID: "ap1", Payload: []byte("ok body"), TerminalState: "approved"}
art, _, err := reg.Request(ctx, facts, req)
if err != nil {
t.Fatal(err)
}
if err := pub.Drain(ctx); err != nil {
t.Fatal(err)
}
_ = reg.Tombstone(ctx, art.UUID, "legal", "admin")
if _, _, err := pub.DegradedPayload(ctx, "dev", "doc", "p"); !errors.Is(err, ErrUnionDeny) {
t.Fatalf("tombstone must union-deny: %v", err)
}
}
func TestLastReferencePurgeAndLegalHold(t *testing.T) {
ctx := context.Background()
reg := NewMemoryRegistry()
store := testStore(t)
pub := &Publisher{Reg: reg, Store: store}
now := time.Now().UTC()
payload := []byte("shared-bytes")
facts := Facts{ArtifactType: TypeSealedVersion, VersionStatus: "sealed", SealedAt: &now, HeadSnapshotID: 2, Classification: ClassificationOK, Payload: payload}
req := PublishRequest{ArtifactType: TypeSealedVersion, Environment: "dev", ProjectExternalKey: "p1", LogicalResourceID: "d1", SourceEventKind: "seal", SourceEventID: "s1", Payload: payload, TerminalState: "sealed"}
if _, _, err := reg.Request(ctx, facts, req); err != nil {
t.Fatal(err)
}
if err := pub.Drain(ctx); err != nil {
t.Fatal(err)
}
if reg.ActiveRefs(ctx, "p1", SHA256Hex(payload)) != 1 {
t.Fatal("expected one active ref")
}
if err := CanPurge(false, 1); err == nil {
t.Fatal("must not purge while refs remain")
}
if err := CanPurge(true, 0); err == nil {
t.Fatal("legal hold blocks purge")
}
if err := CanPurge(false, 0); err != nil {
t.Fatal(err)
}
}
func TestFilesystemIdentityMismatch(t *testing.T) {
dir := t.TempDir()
key := []byte("0123456789abcdef0123456789abcdef")
if _, err := NewFilesystemStore(dir, "id-a", key); err != nil {
t.Fatal(err)
}
if _, err := NewFilesystemStore(dir, "id-b", key); err == nil {
t.Fatal("identity mismatch must fail closed")
}
}
func TestUnpublishedWithoutCommitMarkerInvisible(t *testing.T) {
ctx := context.Background()
reg := NewMemoryRegistry()
store := testStore(t)
pub := &Publisher{Reg: reg, Store: store}
facts := Facts{
ArtifactType: TypeAgentCommit, CandidateStatus: "committed", OutputSnapshotID: 4,
ChangeKind: "agent_commit", ChangeToSnapshotID: 4, Classification: ClassificationOK,
Payload: []byte("# pending\n"),
}
req := PublishRequest{
ArtifactType: TypeAgentCommit, Environment: "dev", ProjectExternalKey: "proj-a",
LogicalResourceID: "doc_pending", SourceEventKind: "agent_commit", SourceEventID: "ev-pending",
Payload: []byte("# pending\n"), TerminalState: "committed",
}
if _, _, err := reg.Request(ctx, facts, req); err != nil {
t.Fatal(err)
}
if _, _, err := pub.DegradedPayload(ctx, "dev", "doc_pending", "proj-a"); err == nil {
t.Fatal("requested but unpublished must be invisible")
}
}
func TestPublisherRetriesAfterMissingCommit(t *testing.T) {
ctx := context.Background()
reg := NewMemoryRegistry()
store := testStore(t)
pub := &Publisher{Reg: reg, Store: store}
payload := []byte("# retry-me\n")
facts := Facts{
ArtifactType: TypeAgentCommit, CandidateStatus: "committed", OutputSnapshotID: 5,
ChangeKind: "agent_commit", ChangeToSnapshotID: 5, Classification: ClassificationOK,
Payload: payload,
}
req := PublishRequest{
ArtifactType: TypeAgentCommit, Environment: "dev", ProjectExternalKey: "proj-a",
LogicalResourceID: "doc_retry", SourceEventKind: "agent_commit", SourceEventID: "ev-retry",
Payload: payload, TerminalState: "committed",
}
art, _, err := reg.Request(ctx, facts, req)
if err != nil {
t.Fatal(err)
}
if err := pub.Drain(ctx); err != nil {
t.Fatal(err)
}
if err := os.RemoveAll(filepath.Join(store.Root, "commits", art.UUID)); err != nil {
t.Fatal(err)
}
if _, _, err := pub.DegradedPayload(ctx, "dev", "doc_retry", "proj-a"); err == nil {
t.Fatal("missing commit marker must hide payload")
}
if err := reg.MarkFailed(ctx, art.UUID, "commit_missing"); err != nil {
t.Fatal(err)
}
if err := pub.Drain(ctx); err != nil {
t.Fatal(err)
}
b, _, err := pub.DegradedPayload(ctx, "dev", "doc_retry", "proj-a")
if err != nil || string(b) != string(payload) {
t.Fatalf("retry after exclusive blob/manifest: %v %q", err, b)
}
}
type requestCountingReg struct {
Registry
requests int
}
func (r *requestCountingReg) Request(ctx context.Context, facts Facts, req PublishRequest) (*Artifact, *Staging, error) {
r.requests++
return r.Registry.Request(ctx, facts, req)
}
func TestDegradedPayloadDoesNotWrite(t *testing.T) {
ctx := context.Background()
mem := NewMemoryRegistry()
count := &requestCountingReg{Registry: mem}
store := testStore(t)
pub := &Publisher{Reg: count, Store: store}
facts := Facts{
ArtifactType: TypeAgentCommit, CandidateStatus: "committed", OutputSnapshotID: 6,
ChangeKind: "agent_commit", ChangeToSnapshotID: 6, Classification: ClassificationOK,
Payload: []byte("# get\n"),
}
req := PublishRequest{
ArtifactType: TypeAgentCommit, Environment: "dev", ProjectExternalKey: "proj-a",
LogicalResourceID: "doc_get", SourceEventKind: "agent_commit", SourceEventID: "ev-get",
Payload: []byte("# get\n"), TerminalState: "committed",
}
if _, _, err := mem.Request(ctx, facts, req); err != nil {
t.Fatal(err)
}
if err := pub.Drain(ctx); err != nil {
t.Fatal(err)
}
count.requests = 0
if _, _, err := pub.DegradedPayload(ctx, "dev", "doc_get", "proj-a"); err != nil {
t.Fatal(err)
}
if count.requests != 0 {
t.Fatalf("GET/degraded read wrote Request %d times", count.requests)
}
}
func ptrTime() *time.Time { t := time.Now().UTC(); return &t }
+139
View File
@@ -0,0 +1,139 @@
package artifactvault
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
)
type RecoverMode string
const (
RecoverAsCollection RecoverMode = "recover-as-collection"
RecoverRelink RecoverMode = "relink"
)
type RecoverPlan struct {
Mode RecoverMode
TargetIdentity string
SourceIdentity string
PlanHash string
Conflicts []string
Actor string
HasAdmin bool
HasProjectAuth bool
DependenciesOK bool
ScratchOnly bool
}
func ValidateRecoverTarget(dsn, identity string) error {
low := strings.ToLower(dsn + " " + identity)
for _, ban := range []string{"agent_system_dev", "agent_system_test", "agent_system_main", "env=dev", "env=test", "env=main"} {
if strings.Contains(low, ban) && !strings.Contains(low, "scratch") {
return fmt.Errorf("artifactvault: refuse recover into operational database")
}
}
if !strings.Contains(low, "scratch") {
return fmt.Errorf("artifactvault: recover target must be scratch")
}
return nil
}
func EvaluateRecoverPlan(p RecoverPlan, denied bool) error {
if err := ValidateRecoverTarget(p.TargetIdentity, p.SourceIdentity); err != nil {
return err
}
if !p.ScratchOnly {
return fmt.Errorf("artifactvault: scratch-only required")
}
if !p.HasAdmin {
return fmt.Errorf("artifactvault: operator lacks role-admin")
}
if denied {
return fmt.Errorf("artifactvault: union-deny deletion ledger")
}
if len(p.Conflicts) > 0 && p.Mode == RecoverRelink {
return fmt.Errorf("artifactvault: relink cannot proceed with conflicts")
}
if p.Mode == RecoverRelink && (!p.HasProjectAuth || !p.DependenciesOK) {
return fmt.Errorf("artifactvault: relink authorization or dependency gap")
}
if p.Mode == "" {
p.Mode = RecoverAsCollection
}
if p.PlanHash == "" {
return fmt.Errorf("artifactvault: plan hash required")
}
return nil
}
type RecoveredItem struct {
ArtifactUUID string `json:"artifact_uuid"`
LogicalResourceID string `json:"logical_resource_id"`
PayloadSHA256 string `json:"payload_sha256"`
SkippedReason string `json:"skipped_reason,omitempty"`
}
type RecoveredCollection struct {
Mode RecoverMode `json:"mode"`
PlanHash string `json:"plan_hash"`
DestDir string `json:"dest_dir"`
Items []RecoveredItem `json:"items"`
}
// ApplyRecoverAsCollection writes recovered payloads into a scratch collection directory.
// It never writes operational databases or relinks live heads.
func ApplyRecoverAsCollection(plan RecoverPlan, destDir string, arts []Artifact, payloads map[string][]byte, denied map[string]bool) (*RecoveredCollection, error) {
if err := EvaluateRecoverPlan(plan, false); err != nil {
return nil, err
}
if plan.Mode != RecoverAsCollection && plan.Mode != "" {
return nil, fmt.Errorf("artifactvault: apply only supports recover-as-collection")
}
if !destIsScratch(destDir) {
return nil, fmt.Errorf("artifactvault: collection dest must be scratch")
}
if err := os.MkdirAll(filepath.Join(destDir, "payloads"), 0o700); err != nil {
return nil, err
}
out := &RecoveredCollection{Mode: RecoverAsCollection, PlanHash: plan.PlanHash, DestDir: destDir}
for _, art := range arts {
item := RecoveredItem{ArtifactUUID: art.UUID, LogicalResourceID: art.LogicalResourceID, PayloadSHA256: art.PayloadSHA256}
if denied[art.UUID] || art.PublicationStatus == StatusTombstoned {
item.SkippedReason = "union-deny"
out.Items = append(out.Items, item)
continue
}
body := payloads[art.UUID]
if len(body) == 0 || SHA256Hex(body) != art.PayloadSHA256 {
item.SkippedReason = "sha_mismatch"
out.Items = append(out.Items, item)
continue
}
path := filepath.Join(destDir, "payloads", art.UUID+".md")
if err := os.WriteFile(path, body, 0o600); err != nil {
return nil, err
}
out.Items = append(out.Items, item)
}
raw, err := json.MarshalIndent(out, "", " ")
if err != nil {
return nil, err
}
if err := os.WriteFile(filepath.Join(destDir, "collection.json"), raw, 0o600); err != nil {
return nil, err
}
return out, nil
}
func destIsScratch(destDir string) bool {
for _, part := range strings.Split(filepath.ToSlash(filepath.Clean(destDir)), "/") {
low := strings.ToLower(part)
if low == "scratch" || strings.HasPrefix(low, "scratch-") || strings.HasPrefix(low, "scratch_") {
return true
}
}
return false
}
+65
View File
@@ -0,0 +1,65 @@
package artifactvault
import (
"os"
"path/filepath"
"testing"
)
func TestRecoverRejectsOperationalDSN(t *testing.T) {
if err := ValidateRecoverTarget("postgres://x/agent_system_dev", "dev"); err == nil {
t.Fatal("dev dsn")
}
if err := ValidateRecoverTarget("postgres://x/fav_scratch", "scratch-1"); err != nil {
t.Fatal(err)
}
}
func TestRecoverPlanGates(t *testing.T) {
p := RecoverPlan{Mode: RecoverAsCollection, TargetIdentity: "scratch-db", SourceIdentity: "vault", PlanHash: "abc", HasAdmin: true, ScratchOnly: true}
if err := EvaluateRecoverPlan(p, false); err != nil {
t.Fatal(err)
}
if err := EvaluateRecoverPlan(p, true); err == nil {
t.Fatal("denied")
}
p.Mode = RecoverRelink
p.Conflicts = []string{"uuid_collision"}
if err := EvaluateRecoverPlan(p, false); err == nil {
t.Fatal("relink+conflict")
}
}
func TestApplyRecoverAsCollectionWritesScratchAndSkipsDenied(t *testing.T) {
dest := filepath.Join(t.TempDir(), "scratch-collection")
plan := RecoverPlan{
Mode: RecoverAsCollection, TargetIdentity: "scratch-db", SourceIdentity: "vault",
PlanHash: "plan-1", HasAdmin: true, ScratchOnly: true,
}
okBody := []byte("# recovered\n")
arts := []Artifact{
{UUID: "art-ok", LogicalResourceID: "doc_ok", PayloadSHA256: SHA256Hex(okBody)},
{UUID: "art-deny", LogicalResourceID: "doc_deny", PayloadSHA256: SHA256Hex([]byte("x")), PublicationStatus: StatusTombstoned},
}
payloads := map[string][]byte{"art-ok": okBody}
out, err := ApplyRecoverAsCollection(plan, dest, arts, payloads, map[string]bool{"art-deny": true})
if err != nil {
t.Fatal(err)
}
if _, err := os.Stat(filepath.Join(dest, "collection.json")); err != nil {
t.Fatal(err)
}
got, err := os.ReadFile(filepath.Join(dest, "payloads", "art-ok.md"))
if err != nil || string(got) != string(okBody) {
t.Fatalf("payload: %v %q", err, got)
}
if _, err := os.Stat(filepath.Join(dest, "payloads", "art-deny.md")); err == nil {
t.Fatal("denied artifact must not be written")
}
if len(out.Items) != 2 || out.Items[1].SkippedReason != "union-deny" {
t.Fatalf("items=%+v", out.Items)
}
if _, err := ApplyRecoverAsCollection(plan, filepath.Join(t.TempDir(), "live-restore"), arts, payloads, nil); err == nil {
t.Fatal("non-scratch dest must fail")
}
}
+467
View File
@@ -0,0 +1,467 @@
package artifactvault
import (
"context"
"fmt"
"sync"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
)
type Registry interface {
Request(ctx context.Context, facts Facts, req PublishRequest) (*Artifact, *Staging, error)
LoadRequested(ctx context.Context) ([]Artifact, error)
LoadStaging(ctx context.Context, sourceEventID string) (*Staging, error)
MarkPublishing(ctx context.Context, uuid string) error
MarkVerified(ctx context.Context, uuid, manifestSHA string) error
MarkFailed(ctx context.Context, uuid, code string) error
GetByLogical(ctx context.Context, env, logical string) (*Artifact, error)
Tombstone(ctx context.Context, uuid, reason, authority string) error
IsDenied(ctx context.Context, uuid string) bool
RecordReference(ctx context.Context, uuid, projectKey, sha, keyRef string) error
ActiveRefs(ctx context.Context, projectKey, sha string) int
Quarantine(ctx context.Context, reason string, logical string) error
QuarantineCount(ctx context.Context) int
Coverage(ctx context.Context) Coverage
}
type Coverage struct {
Required int
Verified int
Quarantine int
Excluded int
}
type MemoryRegistry struct {
mu sync.Mutex
arts map[string]*Artifact
byKey map[string]string
staging map[string]*Staging
denied map[string]bool
refs map[string]int
quarantine int
required int
}
func NewMemoryRegistry() *MemoryRegistry {
return &MemoryRegistry{
arts: map[string]*Artifact{}, byKey: map[string]string{}, staging: map[string]*Staging{},
denied: map[string]bool{}, refs: map[string]int{},
}
}
func (m *MemoryRegistry) Request(_ context.Context, facts Facts, req PublishRequest) (*Artifact, *Staging, error) {
if err := EvaluateEligibility(facts); err != nil {
return nil, nil, err
}
if err := ValidateEnv(req.Environment); err != nil {
return nil, nil, err
}
sha := SHA256Hex(req.Payload)
key := IdempotencyKey(req, sha)
m.mu.Lock()
defer m.mu.Unlock()
if id, ok := m.byKey[key]; ok {
return m.arts[id], m.staging[req.SourceEventID], nil
}
uuid := "art-" + sha[:16] + "-" + fmt.Sprintf("%d", len(m.arts)+1)
art := &Artifact{
UUID: uuid, IdempotencyKey: key, ArtifactType: req.ArtifactType,
LogicalResourceID: req.LogicalResourceID, Environment: req.Environment,
ProjectID: req.ProjectID, ProjectExternalKey: req.ProjectExternalKey,
SourceEventID: req.SourceEventID, SourceEventKind: req.SourceEventKind,
SourceSnapshotID: req.SourceSnapshotID, SourceMessageID: req.SourceMessageID,
SourceRunID: req.SourceRunID, PayloadSHA256: sha, TerminalState: req.TerminalState,
PolicyVersion: nz(req.PolicyVersion, PolicyVersion), PublicationStatus: StatusRequested,
RetentionClass: nz(req.RetentionClass, "governance"), RequestedAt: time.Now().UTC(),
}
st := &Staging{
UUID: "stg-" + uuid, ProjectID: req.ProjectID, SourceEventID: req.SourceEventID,
PayloadSHA256: sha, Payload: append([]byte(nil), req.Payload...), Classification: ClassificationOK,
CreatedAt: time.Now().UTC(),
}
m.arts[uuid] = art
m.byKey[key] = uuid
m.staging[req.SourceEventID] = st
m.required++
return art, st, nil
}
func (m *MemoryRegistry) LoadRequested(_ context.Context) ([]Artifact, error) {
m.mu.Lock()
defer m.mu.Unlock()
out := make([]Artifact, 0)
for _, a := range m.arts {
if a.PublicationStatus == StatusRequested || a.PublicationStatus == StatusRetryableFailed {
out = append(out, *a)
}
}
return out, nil
}
func (m *MemoryRegistry) LoadStaging(_ context.Context, sourceEventID string) (*Staging, error) {
m.mu.Lock()
defer m.mu.Unlock()
st := m.staging[sourceEventID]
if st == nil {
return nil, fmt.Errorf("artifactvault: staging missing")
}
cp := *st
cp.Payload = append([]byte(nil), st.Payload...)
return &cp, nil
}
func (m *MemoryRegistry) MarkPublishing(_ context.Context, uuid string) error {
m.mu.Lock()
defer m.mu.Unlock()
if a := m.arts[uuid]; a != nil {
a.PublicationStatus = StatusPublishing
}
return nil
}
func (m *MemoryRegistry) MarkVerified(_ context.Context, uuid, manifestSHA string) error {
m.mu.Lock()
defer m.mu.Unlock()
if a := m.arts[uuid]; a != nil {
now := time.Now().UTC()
a.PublicationStatus = StatusVerified
a.ManifestSHA256 = manifestSHA
a.VerifiedAt = &now
}
return nil
}
func (m *MemoryRegistry) MarkFailed(_ context.Context, uuid, _ string) error {
m.mu.Lock()
defer m.mu.Unlock()
if a := m.arts[uuid]; a != nil {
a.PublicationStatus = StatusRetryableFailed
}
return nil
}
func (m *MemoryRegistry) GetByLogical(_ context.Context, env, logical string) (*Artifact, error) {
m.mu.Lock()
defer m.mu.Unlock()
for _, a := range m.arts {
if a.Environment == env && a.LogicalResourceID == logical {
cp := *a
return &cp, nil
}
}
return nil, fmt.Errorf("artifactvault: not found")
}
func (m *MemoryRegistry) Tombstone(_ context.Context, uuid, _, _ string) error {
m.mu.Lock()
defer m.mu.Unlock()
m.denied[uuid] = true
if a := m.arts[uuid]; a != nil {
now := time.Now().UTC()
a.PublicationStatus = StatusTombstoned
a.TombstonedAt = &now
}
return nil
}
func (m *MemoryRegistry) IsDenied(_ context.Context, uuid string) bool {
m.mu.Lock()
defer m.mu.Unlock()
return m.denied[uuid]
}
func (m *MemoryRegistry) RecordReference(_ context.Context, _, projectKey, sha, _ string) error {
m.mu.Lock()
defer m.mu.Unlock()
m.refs[projectKey+"|"+sha]++
return nil
}
func (m *MemoryRegistry) ActiveRefs(_ context.Context, projectKey, sha string) int {
m.mu.Lock()
defer m.mu.Unlock()
return m.refs[projectKey+"|"+sha]
}
func (m *MemoryRegistry) Quarantine(_ context.Context, _, _ string) error {
m.mu.Lock()
defer m.mu.Unlock()
m.quarantine++
return nil
}
func (m *MemoryRegistry) QuarantineCount(_ context.Context) int {
m.mu.Lock()
defer m.mu.Unlock()
return m.quarantine
}
func (m *MemoryRegistry) Coverage(_ context.Context) Coverage {
m.mu.Lock()
defer m.mu.Unlock()
v := 0
for _, a := range m.arts {
if a.PublicationStatus == StatusVerified {
v++
}
}
return Coverage{Required: m.required, Verified: v, Quarantine: m.quarantine}
}
// PGRegistry persists FAV rows. Request uses a dedicated transaction unless WithTx is used.
type PGRegistry struct {
pool *pgxpool.Pool
}
func NewPGRegistry(pool *pgxpool.Pool) *PGRegistry { return &PGRegistry{pool: pool} }
func (p *PGRegistry) Request(ctx context.Context, facts Facts, req PublishRequest) (*Artifact, *Staging, error) {
if p == nil || p.pool == nil {
return nil, nil, fmt.Errorf("artifactvault: no database")
}
tx, err := p.pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return nil, nil, err
}
defer func() { _ = tx.Rollback(ctx) }()
art, st, err := requestTX(ctx, tx, facts, req)
if err != nil {
return nil, nil, err
}
if err := tx.Commit(ctx); err != nil {
return nil, nil, err
}
return art, st, nil
}
func requestTX(ctx context.Context, tx pgx.Tx, facts Facts, req PublishRequest) (*Artifact, *Staging, error) {
if err := EvaluateEligibility(facts); err != nil {
return nil, nil, err
}
sha := SHA256Hex(req.Payload)
key := IdempotencyKey(req, sha)
var existing string
err := tx.QueryRow(ctx, `SELECT artifact_uuid FROM final_artifacts WHERE idempotency_key=$1`, key).Scan(&existing)
if err == nil && existing != "" {
st := &Staging{SourceEventID: req.SourceEventID, PayloadSHA256: sha, Payload: req.Payload}
return &Artifact{UUID: existing, IdempotencyKey: key, PublicationStatus: StatusRequested}, st, nil
}
uuid := "art-" + sha[:16]
_, err = tx.Exec(ctx, `
INSERT INTO artifact_payload_staging (staging_uuid, project_id, source_event_id, payload_sha256, encrypted_payload, classification)
VALUES ($1,$2,$3,$4,$5,$6)
ON CONFLICT (source_event_id, payload_sha256) DO NOTHING
`, "stg-"+uuid, req.ProjectID, req.SourceEventID, sha, req.Payload, ClassificationOK)
if err != nil {
return nil, nil, err
}
_, err = tx.Exec(ctx, `
INSERT INTO final_artifacts (
artifact_uuid, idempotency_key, artifact_type, logical_resource_id, environment, project_id,
project_external_key, source_event_id, source_event_kind, source_snapshot_id, source_message_id,
source_run_id, payload_sha256, terminal_state, policy_version, publication_status, retention_class
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17)
ON CONFLICT (idempotency_key) DO NOTHING
`, uuid, key, req.ArtifactType, req.LogicalResourceID, req.Environment, req.ProjectID,
req.ProjectExternalKey, req.SourceEventID, req.SourceEventKind, nullInt(req.SourceSnapshotID),
nullInt(req.SourceMessageID), req.SourceRunID, sha, req.TerminalState, nz(req.PolicyVersion, PolicyVersion),
StatusRequested, nz(req.RetentionClass, "governance"))
if err != nil {
return nil, nil, err
}
return &Artifact{UUID: uuid, IdempotencyKey: key, ArtifactType: req.ArtifactType, LogicalResourceID: req.LogicalResourceID,
Environment: req.Environment, ProjectExternalKey: req.ProjectExternalKey, SourceEventID: req.SourceEventID,
PayloadSHA256: sha, PublicationStatus: StatusRequested, TerminalState: req.TerminalState}, &Staging{Payload: req.Payload, PayloadSHA256: sha, SourceEventID: req.SourceEventID}, nil
}
func nullInt(v int64) any {
if v == 0 {
return nil
}
return v
}
func (p *PGRegistry) LoadRequested(ctx context.Context) ([]Artifact, error) {
if p == nil || p.pool == nil {
return nil, fmt.Errorf("artifactvault: no database")
}
rows, err := p.pool.Query(ctx, `
SELECT artifact_uuid, idempotency_key, artifact_type, logical_resource_id, environment,
COALESCE(project_id,0), project_external_key, source_event_id, source_event_kind,
COALESCE(source_snapshot_id,0), COALESCE(source_message_id,0), source_run_id,
payload_sha256, COALESCE(manifest_sha256,''), COALESCE(vault_storage_identity,''),
terminal_state, policy_version, publication_status, retention_class, requested_at
FROM final_artifacts
WHERE publication_status IN ('requested','retryable_failed')
ORDER BY requested_at ASC`)
if err != nil {
return nil, err
}
defer rows.Close()
out := make([]Artifact, 0)
for rows.Next() {
var a Artifact
if err := rows.Scan(&a.UUID, &a.IdempotencyKey, &a.ArtifactType, &a.LogicalResourceID, &a.Environment,
&a.ProjectID, &a.ProjectExternalKey, &a.SourceEventID, &a.SourceEventKind,
&a.SourceSnapshotID, &a.SourceMessageID, &a.SourceRunID, &a.PayloadSHA256, &a.ManifestSHA256,
&a.VaultStorageIdentity, &a.TerminalState, &a.PolicyVersion, &a.PublicationStatus, &a.RetentionClass, &a.RequestedAt); err != nil {
return nil, err
}
out = append(out, a)
}
return out, rows.Err()
}
func (p *PGRegistry) LoadStaging(ctx context.Context, sourceEventID string) (*Staging, error) {
if p == nil || p.pool == nil {
return nil, fmt.Errorf("artifactvault: no database")
}
var st Staging
err := p.pool.QueryRow(ctx, `
SELECT staging_uuid, COALESCE(project_id,0), source_event_id, payload_sha256, encrypted_payload, classification, created_at
FROM artifact_payload_staging WHERE source_event_id=$1 ORDER BY created_at DESC LIMIT 1`, sourceEventID).
Scan(&st.UUID, &st.ProjectID, &st.SourceEventID, &st.PayloadSHA256, &st.Payload, &st.Classification, &st.CreatedAt)
if err != nil {
return nil, err
}
return &st, nil
}
func (p *PGRegistry) MarkPublishing(ctx context.Context, uuid string) error {
if p == nil || p.pool == nil {
return nil
}
_, err := p.pool.Exec(ctx, `UPDATE final_artifacts SET publication_status=$2 WHERE artifact_uuid=$1`, uuid, StatusPublishing)
if err != nil {
return err
}
_, err = p.pool.Exec(ctx, `
INSERT INTO artifact_publication_attempts (artifact_uuid, idempotency_key, attempt_no)
SELECT artifact_uuid, idempotency_key, COALESCE((SELECT MAX(attempt_no) FROM artifact_publication_attempts a WHERE a.artifact_uuid=$1),0)+1
FROM final_artifacts WHERE artifact_uuid=$1`, uuid)
return err
}
func (p *PGRegistry) MarkVerified(ctx context.Context, uuid, manifestSHA string) error {
if p == nil || p.pool == nil {
return nil
}
_, err := p.pool.Exec(ctx, `UPDATE final_artifacts SET publication_status=$2, manifest_sha256=$3, verified_at=now() WHERE artifact_uuid=$1`, uuid, StatusVerified, manifestSHA)
return err
}
func (p *PGRegistry) MarkFailed(ctx context.Context, uuid, code string) error {
if p == nil || p.pool == nil {
return nil
}
_, err := p.pool.Exec(ctx, `UPDATE final_artifacts SET publication_status=$2 WHERE artifact_uuid=$1`, uuid, StatusRetryableFailed)
if err != nil {
return err
}
_, _ = p.pool.Exec(ctx, `
UPDATE artifact_publication_attempts SET error_code=$2
WHERE artifact_uuid=$1 AND attempt_no=(SELECT MAX(attempt_no) FROM artifact_publication_attempts WHERE artifact_uuid=$1)`, uuid, code)
return nil
}
func (p *PGRegistry) GetByLogical(ctx context.Context, env, logical string) (*Artifact, error) {
if p == nil || p.pool == nil {
return nil, fmt.Errorf("artifactvault: no database")
}
var a Artifact
err := p.pool.QueryRow(ctx, `
SELECT artifact_uuid, idempotency_key, artifact_type, logical_resource_id, environment,
COALESCE(project_id,0), project_external_key, source_event_id, source_event_kind,
COALESCE(source_snapshot_id,0), COALESCE(source_message_id,0), source_run_id,
payload_sha256, COALESCE(manifest_sha256,''), COALESCE(vault_storage_identity,''),
terminal_state, policy_version, publication_status
FROM final_artifacts
WHERE environment=$1 AND logical_resource_id=$2 AND publication_status='verified'
ORDER BY verified_at DESC NULLS LAST LIMIT 1`, env, logical).
Scan(&a.UUID, &a.IdempotencyKey, &a.ArtifactType, &a.LogicalResourceID, &a.Environment,
&a.ProjectID, &a.ProjectExternalKey, &a.SourceEventID, &a.SourceEventKind,
&a.SourceSnapshotID, &a.SourceMessageID, &a.SourceRunID, &a.PayloadSHA256, &a.ManifestSHA256,
&a.VaultStorageIdentity, &a.TerminalState, &a.PolicyVersion, &a.PublicationStatus)
if err != nil {
return nil, err
}
return &a, nil
}
func (p *PGRegistry) Tombstone(ctx context.Context, uuid, reason, authority string) error {
if p == nil || p.pool == nil {
return nil
}
_, err := p.pool.Exec(ctx, `UPDATE final_artifacts SET publication_status=$2, tombstoned_at=now() WHERE artifact_uuid=$1`, uuid, StatusTombstoned)
if err != nil {
return err
}
_, err = p.pool.Exec(ctx, `INSERT INTO artifact_deletion_ledger (artifact_uuid, reason, authority, effective_at) VALUES ($1,$2,$3,now())`, uuid, reason, authority)
return err
}
func (p *PGRegistry) IsDenied(ctx context.Context, uuid string) bool {
if p == nil || p.pool == nil {
return false
}
var n int
_ = p.pool.QueryRow(ctx, `SELECT COUNT(*) FROM artifact_deletion_ledger WHERE artifact_uuid=$1`, uuid).Scan(&n)
return n > 0
}
func (p *PGRegistry) RecordReference(ctx context.Context, uuid, projectKey, sha, keyRef string) error {
if p == nil || p.pool == nil {
return nil
}
_, err := p.pool.Exec(ctx, `
INSERT INTO artifact_blob_references (artifact_uuid, project_scope, payload_sha256, envelope_key_ref, reference_status)
VALUES ($1,$2,$3,$4,'active')
ON CONFLICT (artifact_uuid, payload_sha256) DO NOTHING`, uuid, projectKey, sha, keyRef)
return err
}
func (p *PGRegistry) ActiveRefs(ctx context.Context, projectKey, sha string) int {
if p == nil || p.pool == nil {
return 0
}
var n int
_ = p.pool.QueryRow(ctx, `SELECT COUNT(*) FROM artifact_blob_references WHERE project_scope=$1 AND payload_sha256=$2 AND reference_status='active'`, projectKey, sha).Scan(&n)
return n
}
func (p *PGRegistry) Quarantine(ctx context.Context, reason, logical string) error {
if p == nil || p.pool == nil {
return nil
}
_, err := p.pool.Exec(ctx, `INSERT INTO artifact_quarantine (logical_resource_id, reason) VALUES ($1,$2)`, logical, reason)
return err
}
func (p *PGRegistry) QuarantineCount(ctx context.Context) int {
if p == nil || p.pool == nil {
return 0
}
var n int
_ = p.pool.QueryRow(ctx, `SELECT COUNT(*) FROM artifact_quarantine`).Scan(&n)
return n
}
func (p *PGRegistry) Coverage(ctx context.Context) Coverage {
var c Coverage
if p == nil || p.pool == nil {
return c
}
_ = p.pool.QueryRow(ctx, `SELECT COUNT(*) FROM final_artifacts`).Scan(&c.Required)
_ = p.pool.QueryRow(ctx, `SELECT COUNT(*) FROM final_artifacts WHERE publication_status='verified'`).Scan(&c.Verified)
_ = p.pool.QueryRow(ctx, `SELECT COUNT(*) FROM artifact_quarantine`).Scan(&c.Quarantine)
_ = p.pool.QueryRow(ctx, `SELECT COUNT(*) FROM artifact_backfill_exclusions WHERE expires_at > now()`).Scan(&c.Excluded)
return c
}
// RequestWithTx writes staging + unpublished outbox inside an existing domain transaction.
func RequestWithTx(ctx context.Context, tx pgx.Tx, facts Facts, req PublishRequest) (*Artifact, *Staging, error) {
return requestTX(ctx, tx, facts, req)
}
+208
View File
@@ -0,0 +1,208 @@
package artifactvault
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"strings"
"time"
)
const (
SchemaMajorMinor = "1.0"
PublisherVersion = "fav-publisher/1.0"
PolicyVersion = "fav-eligibility/1.0"
ClassificationOK = "ok"
ClassificationSecret = "secret"
ClassificationRestricted = "restricted"
ClassificationFailed = "classification_failed"
TypeAgentCommit = "agent_commit"
TypeSealedVersion = "sealed_version"
TypeApprovedDocument = "approved_document"
TypeFreeformPublish = "freeform_publish"
TypeAssistantReply = "assistant_reply"
StatusRequested = "requested"
StatusPublishing = "publishing"
StatusVerified = "verified"
StatusRetryableFailed = "retryable_failed"
StatusTombstoned = "tombstoned"
StatusPurged = "purged"
StatusQuarantine = "quarantine"
RefActive = "active"
RefTombstoned = "tombstoned"
RefPurged = "purged"
)
// Facts are the authoritative database facts used for eligibility. HTTP handlers must not invent them.
type Facts struct {
ArtifactType string
CandidateStatus string
OutputSnapshotID int64
ChangeKind string
ChangeToSnapshotID int64
VersionStatus string
SealedAt *time.Time
HeadSnapshotID int64
DocState string
ApprovedAt *time.Time
ReviewSnapshotID int64
HasImmutableSnapshot bool
ExplicitPublish bool
ChatTurnStatus string
CompletedAt *time.Time
AssistantMessageID int64
ExportPolicy string
ExplicitPin bool
PinAfterCompleted bool
Payload []byte
Classification string
}
type PublishRequest struct {
ArtifactType string
Environment string
ProjectID int64
ProjectExternalKey string
LogicalResourceID string
SourceEventKind string
SourceEventID string
SourceSnapshotID int64
SourceMessageID int64
SourceRunID string
Payload []byte
TerminalState string
PolicyVersion string
Classification string
Sensitivity string
RetentionClass string
}
type Artifact struct {
UUID string
IdempotencyKey string
ArtifactType string
LogicalResourceID string
Environment string
ProjectID int64
ProjectExternalKey string
SourceEventID string
SourceEventKind string
SourceSnapshotID int64
SourceMessageID int64
SourceRunID string
PayloadSHA256 string
ManifestSHA256 string
VaultStorageIdentity string
TerminalState string
PolicyVersion string
PublicationStatus string
RetentionClass string
RequestedAt time.Time
VerifiedAt *time.Time
TombstonedAt *time.Time
}
type Staging struct {
UUID string
ProjectID int64
SourceEventID string
PayloadSHA256 string
Payload []byte
Classification string
CreatedAt time.Time
}
type Manifest struct {
ArtifactSchemaVersion string `json:"artifact_schema_version"`
ArtifactUUID string `json:"artifact_uuid"`
ArtifactType string `json:"artifact_type"`
LogicalResourceID string `json:"logical_resource_id"`
Environment string `json:"environment"`
ProjectID int64 `json:"project_id,omitempty"`
ProjectExternalKey string `json:"project_external_key"`
SourceEventID string `json:"source_event_id"`
SourceEventKind string `json:"source_event_kind,omitempty"`
SourceSnapshotID *int64 `json:"source_snapshot_id"`
SourceMessageID *int64 `json:"source_message_id"`
SourceRunID string `json:"source_run_id,omitempty"`
PayloadMembers []PayloadMember `json:"payload_members"`
TerminalEvidence TerminalEvidence `json:"terminal_evidence"`
Lineage map[string]any `json:"lineage,omitempty"`
RetentionClass string `json:"retention_class,omitempty"`
Sensitivity string `json:"sensitivity,omitempty"`
EncryptionKeyRef string `json:"encryption_key_ref,omitempty"`
VaultStorageIdentity string `json:"vault_storage_identity"`
PublisherVersion string `json:"publisher_version,omitempty"`
PublishedAt string `json:"published_at,omitempty"`
ManifestSHA256 string `json:"manifest_sha256"`
}
type PayloadMember struct {
Name string `json:"name"`
MediaType string `json:"media_type"`
ByteSize int `json:"byte_size"`
SHA256 string `json:"sha256"`
Required bool `json:"required"`
}
type TerminalEvidence struct {
State string `json:"state"`
DecidedAt string `json:"decided_at"`
ActorRef string `json:"actor_ref,omitempty"`
PolicyVersion string `json:"policy_version"`
}
type CommitMarker struct {
ArtifactUUID string `json:"artifact_uuid"`
ManifestSHA256 string `json:"manifest_sha256"`
MemberSHAs []string `json:"member_shas"`
VaultStorageIdentity string `json:"vault_storage_identity"`
CommittedAt string `json:"committed_at"`
PublisherVersion string `json:"publisher_version"`
}
type Tombstone struct {
ArtifactUUID string `json:"artifact_uuid"`
EventUUID string `json:"event_uuid"`
Reason string `json:"reason"`
Authority string `json:"authority,omitempty"`
EffectiveAt string `json:"effective_at"`
AuditRef string `json:"audit_ref,omitempty"`
}
func SHA256Hex(b []byte) string {
sum := sha256.Sum256(b)
return hex.EncodeToString(sum[:])
}
func IdempotencyKey(req PublishRequest, payloadSHA string) string {
return strings.Join([]string{
req.ArtifactType,
req.Environment,
req.ProjectExternalKey,
req.LogicalResourceID,
req.SourceEventKind,
req.SourceEventID,
payloadSHA,
nz(req.PolicyVersion, PolicyVersion),
}, "|")
}
func nz(v, def string) string {
if strings.TrimSpace(v) == "" {
return def
}
return v
}
func ValidateEnv(env string) error {
switch env {
case "dev", "test", "main":
return nil
default:
return fmt.Errorf("artifactvault: invalid environment %q", env)
}
}
+101
View File
@@ -2,6 +2,7 @@ package db
import (
"context"
"net/url"
"os"
"path/filepath"
"strings"
@@ -9,6 +10,7 @@ import (
"time"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/pressly/goose/v3"
)
func TestMigrationGate_ExpectedFilesMatchAppliedVersion(t *testing.T) {
@@ -78,3 +80,102 @@ func TestMigration032_AllowsMigrationRepairKinds(t *testing.T) {
t.Fatalf("00032 must allow migration_repair on both constraints, got %s", found)
}
}
func TestGooseM2_00034to00036_DownThenUp(t *testing.T) {
if testing.Short() {
t.Skip("goose down is not part of -short")
}
dsn := SkipIfNoTestDatabaseURL(t)
if strings.Contains(dsn, "5433") || strings.Contains(dsn, "agent_system_dev") ||
(strings.Contains(dsn, "5434") && strings.Contains(dsn, "agent_system_test")) ||
strings.Contains(dsn, "5435") {
t.Fatal("TEST_DATABASE_URL points at a shared env port; refuse agent_system_dev/test/main")
}
migDir := ResolveMigrationsDir()
files, err := ListMigrationFiles(migDir)
if err != nil {
t.Fatal(err)
}
want := []string{
"00034_2026-09-14-workspace-lease-fencing.sql",
"00035_2026-09-14-durable-event-delivery.sql",
"00036_2026-09-14-final-artifact-vault.sql",
}
joined := strings.Join(files, "\n")
for _, name := range want {
if !strings.Contains(joined, name) {
t.Fatalf("missing %s", name)
}
}
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
defer cancel()
admin, err := pgxpool.New(ctx, dsn)
if err != nil {
t.Fatal(err)
}
t.Cleanup(admin.Close)
const isolated = "agent_rel_m2_goose"
_, _ = admin.Exec(ctx, `DROP DATABASE IF EXISTS `+isolated)
if _, err := admin.Exec(ctx, `CREATE DATABASE `+isolated); err != nil {
t.Fatalf("create isolated db: %v", err)
}
t.Cleanup(func() {
_, _ = admin.Exec(context.Background(), `DROP DATABASE IF EXISTS `+isolated)
})
m2dsn := rewriteDatabaseName(dsn, isolated)
pool, err := pgxpool.New(ctx, m2dsn)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err := RunMigrations(ctx, pool, migDir); err != nil {
t.Fatalf("up: %v", err)
}
ver, err := CurrentVersion(ctx, pool)
if err != nil || ver < 36 {
t.Fatalf("version after up: %d %v", ver, err)
}
sqlDB := OpenSQLDB(pool)
defer sqlDB.Close()
if err := goose.SetDialect("postgres"); err != nil {
t.Fatal(err)
}
goose.SetTableName(schemaTable)
if err := goose.DownToContext(ctx, sqlDB, migDir, 33); err != nil {
t.Fatalf("down to 33: %v", err)
}
for _, table := range []string{"event_envelopes", "event_deliveries", "event_dead_letters", "final_artifacts"} {
var n int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM information_schema.tables WHERE table_name=$1`, table).Scan(&n); err != nil {
t.Fatal(err)
}
if n != 0 {
t.Fatalf("%s still present after down", table)
}
}
var col int
if err := pool.QueryRow(ctx, `
SELECT COUNT(*) FROM information_schema.columns
WHERE table_name='document_records' AND column_name='lease_fencing_token'
`).Scan(&col); err != nil {
t.Fatal(err)
}
if col != 0 {
t.Fatal("lease_fencing_token must drop with 00034 Down")
}
if err := RunMigrations(ctx, pool, migDir); err != nil {
t.Fatalf("up after down: %v", err)
}
}
func rewriteDatabaseName(dsn, name string) string {
if u, err := url.Parse(dsn); err == nil && u.Scheme != "" {
u.Path = "/" + name
return u.String()
}
return dsn
}
+12 -1
View File
@@ -58,8 +58,12 @@ type Service struct {
Rev *docrevision.Service
RepoRoot string
Authorize Authorizer
// FavDegradedRead is an extension point only. M1 keeps it false.
FavDegradedRead bool
Vault VaultReader
}
type VaultReader interface {
ReadVerified(ctx context.Context, projectID int64, logicalID string) (markdown string, sha string, err error)
}
func FavDegradedReadEnabled() bool {
@@ -101,6 +105,13 @@ func (s *Service) Get(ctx context.Context, p Principal, ref DocRef) (*ContentVie
}
}
if s.FavDegradedRead && s.Vault != nil {
md, sha, verr := s.Vault.ReadVerified(ctx, rec.ProjectID, rec.DocID)
if verr != nil {
return nil, verr
}
return s.viewFromLedger(ctx, rec, md, sha), nil
}
if s.FavDegradedRead {
return nil, ErrFavNotEnabled
}
+38
View File
@@ -2,6 +2,7 @@ package docquery
import (
"context"
"errors"
"os"
"path/filepath"
"testing"
@@ -165,3 +166,40 @@ func TestFavDegradedReadDefaultOff(t *testing.T) {
t.Fatal("FAV_DEGRADED_READ must default false")
}
}
type countingVault struct {
calls int
err error
}
func (v *countingVault) ReadVerified(context.Context, int64, string) (string, string, error) {
v.calls++
if v.err != nil {
return "", "", v.err
}
return "# from vault\n", "abc", nil
}
func TestFavDegradedReadOnlyAfterAuthorize(t *testing.T) {
ctx := context.Background()
gov := docgov.NewGovernanceEngine(nil)
_, _ = gov.CreateOrGetDocument(ctx, "doc_fav", 9, "intake", "t", "V0.5_feasibility", "", 1, "A", 2, "R", 3, "P")
vault := &countingVault{err: errors.New("artifactvault: union-deny")}
svc := &Service{
Gov: gov, FavDegradedRead: true, Vault: vault,
Authorize: func(context.Context, Principal, *docgov.DocumentRecord) error { return ErrClosed },
}
if _, err := svc.Get(ctx, Principal{UserID: 1}, DocRef{DocID: "doc_fav"}); err != ErrClosed {
t.Fatalf("auth must win: %v", err)
}
if vault.calls != 0 {
t.Fatal("vault must not run before authorization")
}
svc.Authorize = func(context.Context, Principal, *docgov.DocumentRecord) error { return nil }
if _, err := svc.Get(ctx, Principal{UserID: 1}, DocRef{DocID: "doc_fav"}); err == nil {
t.Fatal("union-deny must fail closed")
}
if vault.calls != 1 {
t.Fatalf("calls=%d", vault.calls)
}
}