Agent commits append a validated envelope inside the command unit of work. SKIP LOCKED is only used to claim, and stale fencing plus integrity failures go to quarantine or dead-letter. Co-authored-by: Cursor <cursoragent@cursor.com>
194 lines
6.0 KiB
Go
194 lines
6.0 KiB
Go
package events
|
|
|
|
import (
|
|
"context"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func sample(id string, ver int64, payload string) Envelope {
|
|
body := []byte(payload)
|
|
return Envelope{
|
|
EventID: id, EventName: "document.agent_version_committed", SchemaVersion: EnvelopeSchema,
|
|
Environment: "dev", ProjectID: 9, AggregateType: "document", AggregateID: "doc-a",
|
|
AggregateVersion: ver, ActorRef: "user:1", IdempotencyKey: id, PayloadDigest: Digest(body),
|
|
Producer: "docflow", OccurredAt: time.Now().UTC(), Payload: body,
|
|
}
|
|
}
|
|
|
|
func TestTwoAndFourWorkersNoDoubleOwner(t *testing.T) {
|
|
for _, n := range []int{2, 4} {
|
|
bus := NewMemoryBus()
|
|
if err := bus.Append(context.Background(), sample("e1", 1, "v1"), []string{"proj", "notify"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var wg sync.WaitGroup
|
|
now := time.Now()
|
|
for i := 0; i < n; i++ {
|
|
wg.Add(1)
|
|
go func(i int) {
|
|
defer wg.Done()
|
|
_, _ = bus.Claim("proj", "w"+string(rune('a'+i)), 8, now)
|
|
}(i)
|
|
}
|
|
wg.Wait()
|
|
if n := len(bus.Owners("proj")); n > 1 {
|
|
t.Fatalf("workers=%d double owner %#v", n, bus.Owners("proj"))
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestCrashBeforeAfterAck(t *testing.T) {
|
|
bus := NewMemoryBus()
|
|
_ = bus.Append(context.Background(), sample("e2", 1, "body"), []string{"proj"})
|
|
now := time.Now()
|
|
first, err := bus.Claim("proj", "w1", 1, now)
|
|
if err != nil || len(first) != 1 {
|
|
t.Fatalf("claim: %v %#v", err, first)
|
|
}
|
|
// crash-before-side-effect: drop lease by waiting until expiry
|
|
later := now.Add(3 * time.Second)
|
|
second, err := bus.Claim("proj", "w2", 1, later)
|
|
if err != nil || len(second) != 1 {
|
|
t.Fatalf("reclaim: %v", err)
|
|
}
|
|
if err := bus.Apply("proj", first[0], nil); err != ErrStaleFencing {
|
|
t.Fatalf("expired worker ack: %v", err)
|
|
}
|
|
if err := bus.Apply("proj", second[0], nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if bus.SideEffects("proj", "e2") != 1 {
|
|
t.Fatalf("side=%d", bus.SideEffects("proj", "e2"))
|
|
}
|
|
// crash-after-side-effect/ack: replay is inbox-deduped
|
|
if err := bus.Apply("proj", second[0], nil); err != ErrStaleFencing && bus.SideEffects("proj", "e2") != 1 {
|
|
t.Fatalf("replay: %v side=%d", err, bus.SideEffects("proj", "e2"))
|
|
}
|
|
}
|
|
|
|
func TestGapDoesNotSkipVersion(t *testing.T) {
|
|
bus := NewMemoryBus()
|
|
_ = bus.Append(context.Background(), sample("e-v2", 2, "two"), []string{"proj"})
|
|
now := time.Now()
|
|
got, _ := bus.Claim("proj", "w", 4, now)
|
|
if len(got) != 0 {
|
|
t.Fatalf("must wait for missing version 1, claimed %d", len(got))
|
|
}
|
|
_ = bus.Append(context.Background(), sample("e-v1", 1, "one"), []string{"proj"})
|
|
got, _ = bus.Claim("proj", "w", 4, now)
|
|
if len(got) != 1 || got[0].EventID != "e-v1" {
|
|
t.Fatalf("expected v1 first, got %#v", got)
|
|
}
|
|
if err := bus.Apply("proj", got[0], nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got, _ = bus.Claim("proj", "w", 4, now)
|
|
if len(got) != 1 || got[0].EventID != "e-v2" {
|
|
t.Fatalf("expected v2 next, got %#v", got)
|
|
}
|
|
}
|
|
|
|
func TestQuarantineBadDigestForgedProducerCrossProject(t *testing.T) {
|
|
bus := NewMemoryBus()
|
|
bus.BindConsumer("proj", 9)
|
|
bad := sample("bad-digest", 1, "x")
|
|
bad.PayloadDigest = "deadbeef"
|
|
if err := bus.Append(context.Background(), bad, []string{"proj"}); err != ErrIntegrityFailure {
|
|
t.Fatalf("digest: %v", err)
|
|
}
|
|
if bus.Status("proj", "bad-digest") != "quarantine" {
|
|
t.Fatalf("status=%s", bus.Status("proj", "bad-digest"))
|
|
}
|
|
forged := sample("forged", 1, "x")
|
|
forged.Producer = "forged"
|
|
if err := bus.Append(context.Background(), forged, []string{"proj"}); err != ErrIntegrityFailure {
|
|
t.Fatalf("producer: %v", err)
|
|
}
|
|
cross := sample("cross", 1, "x")
|
|
cross.ProjectID = 1
|
|
cross.Payload = []byte("other-project")
|
|
cross.PayloadDigest = Digest(cross.Payload)
|
|
if cross.ProjectID == 9 {
|
|
t.Fatal("fixture")
|
|
}
|
|
if err := bus.Append(context.Background(), cross, []string{"proj"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
now := time.Now()
|
|
got, err := bus.Claim("proj", "w", 4, now)
|
|
if err != nil || len(got) != 1 || got[0].EventID != "cross" {
|
|
t.Fatalf("claim cross: %v %#v", err, got)
|
|
}
|
|
if err := bus.Apply("proj", got[0], nil); err != ErrQuarantine {
|
|
t.Fatalf("cross-project: %v", err)
|
|
}
|
|
if bus.Status("proj", "cross") != "quarantine" {
|
|
t.Fatalf("status=%s", bus.Status("proj", "cross"))
|
|
}
|
|
}
|
|
|
|
func TestAttempt12GoesDeadLetter(t *testing.T) {
|
|
bus := NewMemoryBus()
|
|
_ = bus.Append(context.Background(), sample("retry", 1, "body"), []string{"proj"})
|
|
now := time.Now()
|
|
transient := errTransient{}
|
|
for i := 0; i < MaxAttempts; i++ {
|
|
got, err := bus.Claim("proj", "w", 1, now.Add(time.Duration(i)*time.Second))
|
|
if err != nil || len(got) != 1 {
|
|
t.Fatalf("claim %d: %v %#v", i, err, got)
|
|
}
|
|
applyErr := bus.Apply("proj", got[0], transient)
|
|
if i < MaxAttempts-1 {
|
|
if applyErr != transient {
|
|
t.Fatalf("attempt %d: %v", i+1, applyErr)
|
|
}
|
|
if bus.Status("proj", "retry") != "pending" {
|
|
t.Fatalf("attempt %d status=%s", i+1, bus.Status("proj", "retry"))
|
|
}
|
|
continue
|
|
}
|
|
if applyErr != transient {
|
|
t.Fatalf("final: %v", applyErr)
|
|
}
|
|
}
|
|
if bus.Status("proj", "retry") != "dead" {
|
|
t.Fatalf("after %d attempts status=%s", MaxAttempts, bus.Status("proj", "retry"))
|
|
}
|
|
}
|
|
|
|
type errTransient struct{}
|
|
|
|
func (errTransient) Error() string { return "transient" }
|
|
|
|
func TestAuthAndIntegrityGoDeadLetter(t *testing.T) {
|
|
bus := NewMemoryBus()
|
|
secret := sample("sec", 1, "api_key=sk-live")
|
|
if err := bus.Append(context.Background(), secret, []string{"proj"}); err != ErrAuthFailure {
|
|
t.Fatalf("auth: %v", err)
|
|
}
|
|
ok := sample("ok", 1, "plain")
|
|
_ = bus.Append(context.Background(), ok, []string{"proj"})
|
|
now := time.Now()
|
|
got, _ := bus.Claim("proj", "w", 1, now)
|
|
if len(got) != 1 {
|
|
t.Fatalf("claim %#v", got)
|
|
}
|
|
if err := bus.Apply("proj", got[0], ErrIntegrityFailure); err != ErrIntegrityFailure {
|
|
t.Fatalf("dead: %v", err)
|
|
}
|
|
if bus.Status("proj", "ok") != "dead" {
|
|
t.Fatalf("status=%s", bus.Status("proj", "ok"))
|
|
}
|
|
}
|
|
|
|
func TestReplayRequiresScope(t *testing.T) {
|
|
if err := AuthorizeReplay(ReplayRequest{}); err != ErrReplayScope {
|
|
t.Fatalf("empty: %v", err)
|
|
}
|
|
if err := AuthorizeReplay(ReplayRequest{Consumer: "proj", DryRun: true, Reason: "ops", Operator: "cursor"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|