Agent-System-Backend/internal/store/account_two_process_pg_test.go

package store

import (
	"bytes"
	"context"
	"database/sql"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"testing"
	"time"
)

// Two independent process opens share one isolated PostgreSQL schema. The
// barrier makes their account transactions genuinely concurrent.
func TestAccountTwoProcessPostgresAdmission(t *testing.T) {
	if mode := os.Getenv("U7_ACCOUNT_CHILD_MODE"); mode != "" {
		childDSN := os.Getenv("U7_ACCOUNT_CHILD_DSN")
		barrier := os.Getenv("U7_ACCOUNT_CHILD_BARRIER")
		st, err := OpenExistingPostgres(childDSN, os.Getenv("U7_ACCOUNT_CHILD_ARTIFACTS"))
		if err != nil {
			t.Fatal(err)
		}
		defer st.Close()
		if err := os.WriteFile(os.Getenv("U7_ACCOUNT_CHILD_READY"), []byte("ready"), 0o600); err != nil {
			t.Fatal(err)
		}
		for i := 0; i < 500; i++ {
			if _, err := os.Stat(barrier); err == nil {
				break
			}
			time.Sleep(5 * time.Millisecond)
		}
		if _, err := os.Stat(barrier); err != nil {
			t.Fatal("two-process barrier missing")
		}
		var outcome AccountOutcome
		switch mode {
		case "create":
			outcome, err = st.CreateActivatedAccount(AccountWrite{Owner: "same", Name: "Same", Purpose: LoginKeyPurposeActivate, CreatedBy: "fixture", Actor: "fixture", Action: "account.create"})
		case "bootstrap":
			outcome, err = st.BootstrapFirstAdminWrite(AccountWrite{Owner: os.Getenv("U7_ACCOUNT_CHILD_OWNER"), Name: "Bootstrap", CreatedBy: "fixture", Actor: "fixture"})
		case "bootstrap-versus-create":
			if os.Getenv("U7_ACCOUNT_CHILD_OWNER") == "bootstrap" {
				outcome, err = st.BootstrapFirstAdminWrite(AccountWrite{Owner: "admin", Name: "Admin", CreatedBy: "fixture", Actor: "fixture"})
			} else {
				outcome, err = st.CreateActivatedAccount(AccountWrite{Owner: "ordinary", Name: "Ordinary", Purpose: LoginKeyPurposeActivate, CreatedBy: "fixture", Actor: "fixture", Action: "account.create"})
			}
		default:
			t.Fatal("unknown child mode")
		}
		if err == nil {
			if outcome.OperationID == "" || outcome.Plain == "" {
				t.Fatal("committed account did not produce a complete receipt")
			}
			fmt.Println("U7_RESULT=success")
			return
		}
		if errors.Is(err, ErrConflict) {
			fmt.Println("U7_RESULT=conflict")
			return
		}
		t.Fatalf("unexpected admission error: %v", err)
		return
	}
	baseDSN := os.Getenv("AGENT_PG_TEST_DSN")
	if baseDSN == "" {
		t.Skip("AGENT_PG_TEST_DSN absent; Dev PG gate runs this against a real isolated schema")
	}
	for _, tc := range []struct {
		mode   string
		owners [2]string
	}{
		{"create", [2]string{"one", "two"}},
		{"bootstrap", [2]string{"admin-one", "admin-two"}},
		{"bootstrap-versus-create", [2]string{"bootstrap", "ordinary"}},
	} {
		t.Run(tc.mode, func(t *testing.T) {
			schema := fmt.Sprintf("u7_audit_%d", time.Now().UnixNano())
			admin, err := sql.Open("pgx", baseDSN)
			if err != nil {
				t.Fatal(err)
			}
			defer admin.Close()
			ensurePGTrgm(t, admin)
			if _, err := admin.Exec(`CREATE SCHEMA ` + schema); err != nil {
				t.Fatal(err)
			}
			defer admin.Exec(`DROP SCHEMA IF EXISTS ` + schema + ` CASCADE`)
			dsn := withTestSearchPath(baseDSN, schema)
			artifacts := filepath.Join(t.TempDir(), "artifacts")
			st, err := OpenPostgres(dsn, artifacts)
			if err != nil {
				t.Fatal(err)
			}
			if err := st.EnsureSystemIdentity("pg-two-process", "pg"); err != nil {
				t.Fatal(err)
			}
			st.Close()
			// Hold the shared admission point so both independent processes
			// are demonstrably contending before either may commit.
			lockKey := "bootstrap:first-admin"
			if tc.mode == "create" {
				lockKey = "account:same"
			}
			lockConn, err := admin.Conn(context.Background())
			if err != nil {
				t.Fatal(err)
			}
			defer lockConn.Close()
			if _, err := lockConn.ExecContext(context.Background(), `SELECT pg_advisory_lock(hashtext($1))`, lockKey); err != nil {
				t.Fatal(err)
			}
			released := false
			release := func() {
				if !released {
					released = true
					_, _ = lockConn.ExecContext(context.Background(), `SELECT pg_advisory_unlock(hashtext($1))`, lockKey)
				}
			}
			defer release()
			barrier := filepath.Join(t.TempDir(), "go")
			readyDir := t.TempDir()
			var children [2]*exec.Cmd
			var logs [2]bytes.Buffer
			for i := range children {
				readyPath := filepath.Join(readyDir, fmt.Sprintf("ready-%d", i))
				cmd := exec.Command(os.Args[0], "-test.run=^TestAccountTwoProcessPostgresAdmission$", "-test.v")
				cmd.Env = append(os.Environ(), "U7_ACCOUNT_CHILD_MODE="+tc.mode, "U7_ACCOUNT_CHILD_DSN="+dsn,
					"U7_ACCOUNT_CHILD_OWNER="+tc.owners[i], "U7_ACCOUNT_CHILD_ARTIFACTS="+artifacts,
					"U7_ACCOUNT_CHILD_BARRIER="+barrier, "U7_ACCOUNT_CHILD_READY="+readyPath)
				cmd.Stdout = &logs[i]
				cmd.Stderr = &logs[i]
				if err := cmd.Start(); err != nil {
					t.Fatal(err)
				}
				children[i] = cmd
			}
			deadline := time.Now().Add(10 * time.Second)
			for i := range children {
				readyPath := filepath.Join(readyDir, fmt.Sprintf("ready-%d", i))
				for {
					if _, err := os.Stat(readyPath); err == nil {
						break
					}
					if time.Now().After(deadline) {
						t.Fatalf("child %d never opened the shared database: %s", i, logs[i].String())
					}
					time.Sleep(5 * time.Millisecond)
				}
			}
			if err := os.WriteFile(barrier, []byte("go"), 0o600); err != nil {
				t.Fatal(err)
			}
			for {
				var waiters int
				err := admin.QueryRow(`SELECT count(*) FROM pg_locks WHERE locktype='advisory' AND NOT granted
					AND objid=(hashtext($1)::bigint & 4294967295)::oid`, lockKey).Scan(&waiters)
				if err != nil {
					t.Fatal(err)
				}
				if waiters >= 2 {
					break
				}
				if time.Now().After(deadline) {
					t.Fatalf("only %d of 2 child processes reached the shared admission lock: %s | %s", waiters, logs[0].String(), logs[1].String())
				}
				time.Sleep(5 * time.Millisecond)
			}
			release()
			wins, losses := 0, 0
			for i, cmd := range children {
				if err := cmd.Wait(); err != nil {
					t.Fatalf("child %d failed: %v %s", i, err, logs[i].String())
				}
				out := logs[i].String()
				if strings.Contains(out, "U7_RESULT=success") {
					wins++
				} else if strings.Contains(out, "U7_RESULT=conflict") {
					losses++
				} else {
					t.Fatalf("child %d produced no outcome: %s", i, out)
				}
			}
			if wins != 1 || losses != 1 {
				t.Fatalf("admission wins=%d conflicts=%d, want 1/1", wins, losses)
			}
			reopened, err := OpenExistingPostgres(dsn, artifacts)
			if err != nil {
				t.Fatal(err)
			}
			defer reopened.Close()
			users, err := reopened.ListUsersAdmin()
			if err != nil || len(users) != 1 {
				t.Fatalf("actual committed users=%d, err=%v", len(users), err)
			}
		})
	}
}