diff --git a/internal/jobs/backup_extra_test.go b/internal/jobs/backup_extra_test.go
new file mode 100644
index 0000000..5bb6c5e
--- /dev/null
+++ b/internal/jobs/backup_extra_test.go
@@ -0,0 +1,2457 @@
+package jobs
+
+// backup_extra_test.go — coverage-raising tests for the backup job family.
+// Targets the uncovered surface in:
+//
+// * backup_s3.go — NewMinIOBackupStore branches +
+// *minioBackupStore.{Upload,Download,DeleteObject}
+// + commonPlanRegistryAdapter +
+// NewBackupPlanRegistry
+// * customer_backup_runner.go — Kind / Run / NewCustomerBackupRunner /
+// WithRefundClient / limitedBuffer.{Write,String} /
+// refundManualBackupQuota /
+// signBackupRefundJWT + Work edge cases
+// * customer_backup_scheduler.go — Kind / Run
+// * customer_restore_runner.go — Kind / Run / NewCustomerRestoreRunner +
+// download error / connURL empty / bad AES key /
+// decrypt failure / gzip header invalid /
+// recoverStuckRestores success path
+// * platform_db_backup.go — Kind / NewPlatformDBBackupWorker default
+// branches / joinPlatformBackupPrefix edges /
+// Work success-but-list-fail / writeAudit nil DB /
+// defaultPgDumpExec.Dump via PG_DUMP_BIN trick
+// * platform_db_backup_s3.go — NewBackupS3Client branches + minioS3.{Upload,List,Delete}
+//
+// All tests are hermetic — no external Postgres / S3 / network egress.
+// httptest stands in for an S3 endpoint where the production code dials
+// the minio-go SDK. We don't validate the S3 protocol — we validate that
+// our wrappers translate Go calls into HTTP requests + parse the
+// response shape the SDK expects.
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/base64"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "hash"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "runtime"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ sqlmock "github.com/DATA-DOG/go-sqlmock"
+ "github.com/google/uuid"
+ minio "github.com/minio/minio-go/v7"
+
+ commonplans "instant.dev/common/plans"
+)
+
+// ────────────────────────────────────────────────────────────────────
+// backup_s3.go — NewMinIOBackupStore + commonPlanRegistryAdapter
+// ────────────────────────────────────────────────────────────────────
+
+// TestNewMinIOBackupStore_EmptyEndpoint — fail-loud on missing endpoint.
+func TestNewMinIOBackupStore_EmptyEndpoint(t *testing.T) {
+ if _, err := NewMinIOBackupStore("", "k", "s"); err == nil {
+ t.Fatal("expected error for empty endpoint, got nil")
+ }
+}
+
+// TestNewMinIOBackupStore_SchemeAndVendorBranches — exercise every arm of
+// the TLS heuristic so the constructor's switch is fully covered.
+func TestNewMinIOBackupStore_SchemeAndVendorBranches(t *testing.T) {
+ cases := []struct {
+ name string
+ endpoint string
+ }{
+ {"plain_https_prefix", "https://example.com"},
+ {"plain_http_prefix", "http://example.com"},
+ {"vendor_do_spaces", "nyc3.digitaloceanspaces.com"},
+ {"vendor_aws", "s3.us-east-1.amazonaws.com"},
+ {"vendor_cf_r2", "abc.r2.cloudflarestorage.com"},
+ {"vendor_gcs", "storage.googleapis.com"},
+ {"vendor_wasabi", "s3.wasabisys.com"},
+ {"vendor_b2", "s3.us-west-001.backblazeb2.com"},
+ {"unrecognised_no_scheme", "localhost:9000"},
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ store, err := NewMinIOBackupStore(c.endpoint, "key", "secret")
+ if err != nil {
+ t.Fatalf("ctor returned %v", err)
+ }
+ if store == nil || store.client == nil {
+ t.Fatalf("store.client is nil for %s", c.endpoint)
+ }
+ })
+ }
+}
+
+// TestNewBackupPlanRegistry_NilReturnsNil — defensive nil guard.
+func TestNewBackupPlanRegistry_NilReturnsNil(t *testing.T) {
+ if got := NewBackupPlanRegistry(nil); got != nil {
+ t.Fatalf("nil registry should return nil adapter, got %v", got)
+ }
+}
+
+// TestCommonPlanRegistryAdapter_Delegates — the wrapper proxies
+// BackupRetentionDays + TierNames through to the embedded common/plans.Registry.
+func TestCommonPlanRegistryAdapter_Delegates(t *testing.T) {
+ reg := commonplans.Default()
+ if reg == nil {
+ t.Fatal("commonplans.Default returned nil")
+ }
+ adapter := NewBackupPlanRegistry(reg)
+ if adapter == nil {
+ t.Fatal("adapter is nil")
+ }
+ // BackupRetentionDays sanity — pro tier is documented at 30 days
+ // in plans.yaml; defensive: anything > 0 is enough to prove the
+ // delegation works (and that we are not hitting the legacy 7-day
+ // fallback path).
+ if d := adapter.BackupRetentionDays("pro"); d <= 0 {
+ t.Errorf("BackupRetentionDays(pro) = %d; want > 0", d)
+ }
+ names := adapter.TierNames()
+ if len(names) == 0 {
+ t.Fatal("TierNames returned empty slice")
+ }
+ // Must include the major paid tiers — the retention sweep loops this.
+ want := map[string]bool{"hobby": false, "pro": false, "team": false}
+ for _, n := range names {
+ if _, ok := want[n]; ok {
+ want[n] = true
+ }
+ }
+ for tier, seen := range want {
+ if !seen {
+ t.Errorf("TierNames missing %q", tier)
+ }
+ }
+}
+
+// ────────────────────────────────────────────────────────────────────
+// minioBackupStore — Upload / Download / DeleteObject against an
+// httptest server posing as an S3-compatible endpoint.
+// ────────────────────────────────────────────────────────────────────
+
+// newMinIOForHTTPTest dials minio-go at the supplied test-server endpoint.
+// Returns the *minioBackupStore so we can call its methods directly.
+func newMinIOForHTTPTest(t *testing.T, ts *httptest.Server) *minioBackupStore {
+ t.Helper()
+ // Strip "http://" so the minio.New parser doesn't double up.
+ endpoint := strings.TrimPrefix(ts.URL, "http://")
+ cli, err := minio.New(endpoint, &minio.Options{
+ // Inline creds — the httptest handler won't validate them.
+ Creds: nil,
+ Secure: false,
+ })
+ if err != nil {
+ t.Fatalf("minio.New: %v", err)
+ }
+ return &minioBackupStore{client: cli}
+}
+
+// TestMinioBackupStore_Upload_ServerError — wraps minio-go errors with
+// the "backup_s3.Upload" prefix. Hitting a 500 is sufficient to exercise
+// the error-wrap path; we skip the happy-path "200 OK with ETag" test
+// because minio-go's SigV4 signer expects an S3-compliant response shape
+// (xml-encoded result + canonical headers) that a 4-line httptest stub
+// doesn't satisfy — driving the success path requires a real S3 server.
+func TestMinioBackupStore_Upload_ServerError(t *testing.T) {
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusInternalServerError)
+ _, _ = io.WriteString(w, `InternalError`)
+ }))
+ defer ts.Close()
+ store := newMinIOForHTTPTest(t, ts)
+ _, err := store.Upload(context.Background(), "bucket", "key", strings.NewReader("x"))
+ if err == nil {
+ t.Fatal("expected error from 500, got nil")
+ }
+ if !strings.Contains(err.Error(), "backup_s3.Upload") {
+ t.Errorf("error not wrapped: %v", err)
+ }
+}
+
+// TestMinioBackupStore_Download_Returns_ReadCloser — GetObject is lazy
+// in minio-go; the SDK returns an *Object without dialing. We just verify
+// the call returns a non-nil ReadCloser without an error.
+func TestMinioBackupStore_Download_Returns_ReadCloser(t *testing.T) {
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ _, _ = w.Write([]byte("data"))
+ }))
+ defer ts.Close()
+ store := newMinIOForHTTPTest(t, ts)
+ rc, err := store.Download(context.Background(), "bucket", "key")
+ if err != nil {
+ t.Fatalf("Download: %v", err)
+ }
+ if rc == nil {
+ t.Fatal("nil ReadCloser")
+ }
+ _ = rc.Close()
+}
+
+// TestMinioBackupStore_DeleteObject_ServerError — exercise the error-wrap
+// path. (The success path requires a real S3 server — see Upload note.)
+func TestMinioBackupStore_DeleteObject_ServerError(t *testing.T) {
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusInternalServerError)
+ }))
+ defer ts.Close()
+ store := newMinIOForHTTPTest(t, ts)
+ if err := store.DeleteObject(context.Background(), "bucket", "key"); err == nil {
+ t.Fatal("expected error on 500")
+ }
+}
+
+// ────────────────────────────────────────────────────────────────────
+// platform_db_backup_s3.go — NewBackupS3Client + minioS3.{Upload,List,Delete}
+// ────────────────────────────────────────────────────────────────────
+
+// TestNewBackupS3Client_EmptyEndpoint — fails loudly.
+func TestNewBackupS3Client_EmptyEndpoint(t *testing.T) {
+ if _, err := NewBackupS3Client("", "k", "s"); err == nil {
+ t.Fatal("expected error for empty endpoint")
+ }
+}
+
+// TestNewBackupS3Client_AllSchemes — covers each branch of the TLS heuristic.
+func TestNewBackupS3Client_AllSchemes(t *testing.T) {
+ cases := []string{
+ "https://example.com",
+ "http://example.com",
+ "nyc3.digitaloceanspaces.com",
+ "s3.amazonaws.com",
+ "x.r2.cloudflarestorage.com",
+ "storage.googleapis.com",
+ "s3.wasabisys.com",
+ "s3.backblazeb2.com",
+ "localhost:9000",
+ }
+ for _, ep := range cases {
+ t.Run(ep, func(t *testing.T) {
+ cli, err := NewBackupS3Client(ep, "k", "s")
+ if err != nil {
+ t.Fatalf("ctor: %v", err)
+ }
+ if cli == nil {
+ t.Fatal("nil client")
+ }
+ })
+ }
+}
+
+// newMinioS3ForHTTPTest dials minio-go at the supplied httptest server.
+func newMinioS3ForHTTPTest(t *testing.T, ts *httptest.Server) *minioS3 {
+ t.Helper()
+ endpoint := strings.TrimPrefix(ts.URL, "http://")
+ cli, err := minio.New(endpoint, &minio.Options{Secure: false})
+ if err != nil {
+ t.Fatalf("minio.New: %v", err)
+ }
+ return &minioS3{client: cli}
+}
+
+// TestMinioS3_Upload_Errors — 500 propagates as a "PutObject" wrap.
+func TestMinioS3_Upload_Errors(t *testing.T) {
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusInternalServerError)
+ }))
+ defer ts.Close()
+ cli := newMinioS3ForHTTPTest(t, ts)
+ if err := cli.Upload(context.Background(), "bucket", "key", strings.NewReader("x"), 1); err == nil {
+ t.Fatal("expected error, got nil")
+ }
+}
+
+// TestMinioS3_List_Succeeds — minimal ListObjectsV2 response shape.
+func TestMinioS3_List_Succeeds(t *testing.T) {
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ // minio-go reads via ListObjectsV2 — return an empty result so the
+ // channel closes immediately.
+ w.Header().Set("Content-Type", "application/xml")
+ _, _ = io.WriteString(w, `
+
+ bucket
+ p/
+ 0
+ 1000
+ false
+`)
+ }))
+ defer ts.Close()
+ cli := newMinioS3ForHTTPTest(t, ts)
+ keys, err := cli.List(context.Background(), "bucket", "p/")
+ if err != nil {
+ t.Fatalf("List: %v", err)
+ }
+ _ = keys
+}
+
+// TestMinioS3_List_Errors — 500 surfaces a wrapped error.
+func TestMinioS3_List_Errors(t *testing.T) {
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusInternalServerError)
+ }))
+ defer ts.Close()
+ cli := newMinioS3ForHTTPTest(t, ts)
+ _, err := cli.List(context.Background(), "bucket", "p/")
+ if err == nil {
+ t.Fatal("expected error, got nil")
+ }
+}
+
+// TestMinioS3_Delete_Errors — 500 surfaces a wrapped error. (Happy path
+// requires a real S3-compliant 204 — see Upload note for why we don't
+// drive it via httptest here.)
+func TestMinioS3_Delete_Errors(t *testing.T) {
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusInternalServerError)
+ }))
+ defer ts.Close()
+ cli := newMinioS3ForHTTPTest(t, ts)
+ if err := cli.Delete(context.Background(), "bucket", "key"); err == nil {
+ t.Fatal("expected error on 500")
+ }
+}
+
+// ────────────────────────────────────────────────────────────────────
+// customer_backup_runner.go — public surface coverage
+// ────────────────────────────────────────────────────────────────────
+
+// TestCustomerBackupRunnerArgs_KindAndRun — pin the River job key + the
+// (CustomerBackupRunnerArgs).Run helper which is the constructor used
+// by River when scheduling the job.
+func TestCustomerBackupRunnerArgs_KindAndRun(t *testing.T) {
+ args := CustomerBackupRunnerArgs{}
+ if got := args.Kind(); got != "customer_backup_runner" {
+ t.Errorf("Kind() = %q; want customer_backup_runner", got)
+ }
+}
+
+// TestNewCustomerBackupRunner_Defaults — happy ctor wiring.
+func TestNewCustomerBackupRunner_Defaults(t *testing.T) {
+ db, _, err := sqlmock.New()
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ w := NewCustomerBackupRunner(db, newFakeBackupStore(), "bucket", "prefix", testAESKeyHex, nil)
+ if w == nil {
+ t.Fatal("nil worker")
+ }
+ if w.bucket != "bucket" || w.prefix != "prefix" || w.aesKey != testAESKeyHex {
+ t.Errorf("ctor fields not propagated: %+v", w)
+ }
+ if w.timeout != backupPerRunTimeout {
+ t.Errorf("default timeout = %v; want %v", w.timeout, backupPerRunTimeout)
+ }
+ if w.batchN != backupBatchSize {
+ t.Errorf("default batch size = %d; want %d", w.batchN, backupBatchSize)
+ }
+ if w.now == nil {
+ t.Error("now func not initialised")
+ }
+}
+
+// TestWithRefundClient_PopulatesFields — wires apiBase/jwtSecret/apiCli.
+func TestWithRefundClient_PopulatesFields(t *testing.T) {
+ db, _, _ := sqlmock.New()
+ defer db.Close()
+ w := NewCustomerBackupRunner(db, newFakeBackupStore(), "bucket", "prefix", testAESKeyHex, nil)
+ w2 := w.WithRefundClient("https://api.example.com/", "secret", nil)
+ // Returns the same worker so the call chains.
+ if w2 != w {
+ t.Errorf("WithRefundClient should return the same worker")
+ }
+ if w.apiBase != "https://api.example.com" {
+ t.Errorf("apiBase trailing-slash strip failed: %q", w.apiBase)
+ }
+ if w.jwtSecret != "secret" {
+ t.Errorf("jwtSecret not set")
+ }
+ if w.apiCli == nil {
+ t.Errorf("apiCli not set")
+ }
+ // Calling with explicit non-nil http.Client still wires apiCli.
+ w3 := w.WithRefundClient("https://b/", "s2", &http.Client{Timeout: 5 * time.Second})
+ if w3.apiCli == nil {
+ t.Errorf("apiCli nil when explicit http.Client passed")
+ }
+}
+
+// TestLimitedBuffer_WriteAndString — pure helper. Asserts:
+// - first write below cap stores all bytes
+// - oversized write truncates: n returned == bytes that actually landed
+// - once full, additional Write returns len(p) without writing anything
+// (silent drop — matches the comment in customer_backup_runner.go)
+func TestLimitedBuffer_WriteAndString(t *testing.T) {
+ var b limitedBuffer
+ n, err := b.Write([]byte("hello"))
+ if err != nil || n != 5 {
+ t.Fatalf("first write: n=%d err=%v", n, err)
+ }
+ // Fill until truncation. 5000-byte chunk → 4091 actually land (4096 - 5).
+ big := make([]byte, 5000)
+ for i := range big {
+ big[i] = 'X'
+ }
+ n2, err := b.Write(big)
+ if err != nil {
+ t.Fatalf("second write: %v", err)
+ }
+ if n2 != 4091 {
+ t.Errorf("second write n=%d; want 4091 (4096-5 cap)", n2)
+ }
+ s := b.String()
+ if !strings.HasPrefix(s, "helloXX") {
+ t.Errorf("buffer prefix wrong: %q", s[:20])
+ }
+ if len(s) != 4096 {
+ t.Errorf("buffer length = %d; want 4096", len(s))
+ }
+ // Once full, additional writes drop silently — Write returns len(p)
+ // without any bytes hitting the array.
+ n3, err := b.Write([]byte("more"))
+ if err != nil || n3 != 4 {
+ t.Errorf("post-fill write: n=%d err=%v; want n=4, no error (silent drop)", n3, err)
+ }
+ if len(b.String()) != 4096 {
+ t.Errorf("post-fill String length grew: %d", len(b.String()))
+ }
+}
+
+// TestSignBackupRefundJWT_Shape — three-segment HS256 token whose claims
+// JSON contains purpose/team_id/iat/exp.
+func TestSignBackupRefundJWT_Shape(t *testing.T) {
+ tok, err := signBackupRefundJWT("topsecret", "team-uuid")
+ if err != nil {
+ t.Fatalf("signBackupRefundJWT: %v", err)
+ }
+ parts := strings.Split(tok, ".")
+ if len(parts) != 3 {
+ t.Fatalf("token segments = %d; want 3", len(parts))
+ }
+ claimsRaw, err := base64.RawURLEncoding.DecodeString(parts[1])
+ if err != nil {
+ t.Fatalf("base64 claims: %v", err)
+ }
+ var claims map[string]any
+ if err := json.Unmarshal(claimsRaw, &claims); err != nil {
+ t.Fatalf("claims json: %v", err)
+ }
+ if claims["purpose"] != "internal_backup_refund" {
+ t.Errorf("purpose = %v; want internal_backup_refund", claims["purpose"])
+ }
+ if claims["team_id"] != "team-uuid" {
+ t.Errorf("team_id = %v", claims["team_id"])
+ }
+ if _, ok := claims["iat"]; !ok {
+ t.Error("missing iat claim")
+ }
+ if _, ok := claims["exp"]; !ok {
+ t.Error("missing exp claim")
+ }
+}
+
+// TestSignBackupRefundJWT_EmptySecret — defensive error path.
+func TestSignBackupRefundJWT_EmptySecret(t *testing.T) {
+ if _, err := signBackupRefundJWT("", "team"); err == nil {
+ t.Fatal("expected error for empty secret")
+ }
+}
+
+// TestRefundManualBackupQuota_NoConfig_NoOp — apiBase or jwtSecret empty
+// returns nil immediately.
+func TestRefundManualBackupQuota_NoConfig_NoOp(t *testing.T) {
+ db, _, _ := sqlmock.New()
+ defer db.Close()
+ w := NewCustomerBackupRunner(db, newFakeBackupStore(), "b", "p", testAESKeyHex, nil)
+ // Neither WithRefundClient nor manual field set → no-op path.
+ if err := w.refundManualBackupQuota(uuid.New(), "bk"); err != nil {
+ t.Errorf("expected nil from disabled refund, got %v", err)
+ }
+}
+
+// TestRefundManualBackupQuota_2xx_Success — exercise the full HTTP path
+// with a fake api responder that returns 200.
+func TestRefundManualBackupQuota_2xx_Success(t *testing.T) {
+ gotPath := ""
+ gotAuth := ""
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ gotPath = r.URL.Path
+ gotAuth = r.Header.Get("Authorization")
+ _, _ = io.WriteString(w, `{"ok":true}`)
+ }))
+ defer srv.Close()
+ db, _, _ := sqlmock.New()
+ defer db.Close()
+ w := NewCustomerBackupRunner(db, newFakeBackupStore(), "b", "p", testAESKeyHex, nil).
+ WithRefundClient(srv.URL, "secret", nil)
+ team := uuid.New()
+ if err := w.refundManualBackupQuota(team, "bk-1"); err != nil {
+ t.Fatalf("refundManualBackupQuota: %v", err)
+ }
+ wantPath := "/internal/teams/" + team.String() + "/backup-quota/refund"
+ if gotPath != wantPath {
+ t.Errorf("path = %q; want %q", gotPath, wantPath)
+ }
+ if !strings.HasPrefix(gotAuth, "Bearer ") {
+ t.Errorf("missing Bearer prefix: %q", gotAuth)
+ }
+}
+
+// TestRefundManualBackupQuota_4xx_Error — non-2xx body is captured.
+func TestRefundManualBackupQuota_4xx_Error(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusBadRequest)
+ _, _ = io.WriteString(w, `bad`)
+ }))
+ defer srv.Close()
+ db, _, _ := sqlmock.New()
+ defer db.Close()
+ w := NewCustomerBackupRunner(db, newFakeBackupStore(), "b", "p", testAESKeyHex, nil).
+ WithRefundClient(srv.URL, "secret", nil)
+ err := w.refundManualBackupQuota(uuid.New(), "bk-1")
+ if err == nil {
+ t.Fatal("expected error from 400, got nil")
+ }
+ if !strings.Contains(err.Error(), "400") {
+ t.Errorf("error missing status: %v", err)
+ }
+}
+
+// TestRefundManualBackupQuota_NetworkError — connection refused after
+// the test server has shut down. Asserts the network branch surfaces an
+// error rather than panicking.
+func TestRefundManualBackupQuota_NetworkError(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {}))
+ url := srv.URL
+ srv.Close() // shut down immediately so the next dial fails
+ db, _, _ := sqlmock.New()
+ defer db.Close()
+ w := NewCustomerBackupRunner(db, newFakeBackupStore(), "b", "p", testAESKeyHex, nil).
+ WithRefundClient(url, "secret", &http.Client{Timeout: 200 * time.Millisecond})
+ if err := w.refundManualBackupQuota(uuid.New(), "bk-1"); err == nil {
+ t.Fatal("expected dial-error, got nil")
+ }
+}
+
+// ────────────────────────────────────────────────────────────────────
+// customer_backup_runner.go — Work / processBackup edge branches
+// ────────────────────────────────────────────────────────────────────
+
+// TestRunner_Work_SelectError_ReturnsError — DB outage on the pending-row
+// SELECT bubbles up as a Work-level error.
+func TestRunner_Work_SelectError_ReturnsError(t *testing.T) {
+ db, mock, err := sqlmock.New()
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'pending'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT b.id::text`).
+ WithArgs(backupBatchSize).
+ WillReturnError(errors.New("db gone"))
+
+ w := &CustomerBackupRunnerWorker{
+ db: db,
+ store: newFakeBackupStore(),
+ pgDump: &fakePgDump{},
+ bucket: "b",
+ prefix: "p",
+ aesKey: testAESKeyHex,
+ now: time.Now,
+ timeout: time.Minute,
+ batchN: backupBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRunnerJob()); err == nil {
+ t.Fatal("expected error, got nil")
+ }
+}
+
+// (Removed TestRunner_Work_ContextCancelledMidBatch: a pre-cancelled ctx
+// fails the SELECT before the per-row ctx.Done() check is reachable; the
+// happy-path coverage in TestRunner_HappyPath + TestRunner_PgDumpFails_*
+// already exercises the body of the per-row loop.)
+
+// TestRunner_ProcessBackup_DecryptFails — connection_url ciphertext that
+// can't be decrypted (wrong key) marks the row failed.
+func TestRunner_ProcessBackup_DecryptFails(t *testing.T) {
+ db, mock, err := sqlmock.New()
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+
+ backupID := "11111111-1111-1111-1111-111111111111"
+ resID := "22222222-2222-2222-2222-222222222222"
+ teamID := uuid.New()
+
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'pending'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT b.id::text`).
+ WithArgs(backupBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "tier_at_backup", "backup_kind",
+ "token", "connection_url", "resource_type", "team_id",
+ }).AddRow(backupID, resID, "pro", "scheduled", "tk", "GARBAGE-CIPHERTEXT", "postgres", teamID))
+ mock.ExpectQuery(`UPDATE resource_backups\s+SET status = 'running'`).
+ WithArgs(backupID).
+ WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(backupID))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ // Decrypt failure → markFailed UPDATE + audit row.
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'failed'`).
+ WithArgs(backupID, sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ for i := 0; i < 5; i++ {
+ mock.ExpectQuery(`SELECT id::text, s3_key`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "s3_key"}))
+ }
+
+ w := &CustomerBackupRunnerWorker{
+ db: db,
+ store: newFakeBackupStore(),
+ pgDump: &fakePgDump{},
+ bucket: "b",
+ prefix: "p",
+ aesKey: testAESKeyHex,
+ now: time.Now,
+ timeout: time.Minute,
+ batchN: backupBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRunnerJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+// TestRunner_ProcessBackup_EmptyConnURL — NULL/empty connection_url is a
+// hard failure for the row.
+func TestRunner_ProcessBackup_EmptyConnURL(t *testing.T) {
+ db, mock, err := sqlmock.New()
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+
+ backupID := "11111111-1111-1111-1111-111111111111"
+ resID := "22222222-2222-2222-2222-222222222222"
+ teamID := uuid.New()
+
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'pending'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT b.id::text`).
+ WithArgs(backupBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "tier_at_backup", "backup_kind",
+ "token", "connection_url", "resource_type", "team_id",
+ }).AddRow(backupID, resID, "pro", "scheduled", "tk", nil, "postgres", teamID))
+ mock.ExpectQuery(`UPDATE resource_backups\s+SET status = 'running'`).
+ WithArgs(backupID).
+ WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(backupID))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'failed'`).
+ WithArgs(backupID, sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ for i := 0; i < 5; i++ {
+ mock.ExpectQuery(`SELECT id::text, s3_key`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "s3_key"}))
+ }
+
+ w := &CustomerBackupRunnerWorker{
+ db: db, store: newFakeBackupStore(), pgDump: &fakePgDump{},
+ bucket: "b", prefix: "p", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: backupBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRunnerJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+// TestRunner_ProcessBackup_BadAESKey — invalid hex AES key fails the row.
+func TestRunner_ProcessBackup_BadAESKey(t *testing.T) {
+ db, mock, err := sqlmock.New()
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+
+ backupID := "11111111-1111-1111-1111-111111111111"
+ resID := "22222222-2222-2222-2222-222222222222"
+ teamID := uuid.New()
+ enc := encryptForTest(t, "postgres://u:p@host/db")
+
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'pending'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT b.id::text`).
+ WithArgs(backupBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "tier_at_backup", "backup_kind",
+ "token", "connection_url", "resource_type", "team_id",
+ }).AddRow(backupID, resID, "pro", "scheduled", "tk", enc, "postgres", teamID))
+ mock.ExpectQuery(`UPDATE resource_backups\s+SET status = 'running'`).
+ WithArgs(backupID).
+ WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(backupID))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'failed'`).
+ WithArgs(backupID, sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ for i := 0; i < 5; i++ {
+ mock.ExpectQuery(`SELECT id::text, s3_key`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "s3_key"}))
+ }
+
+ w := &CustomerBackupRunnerWorker{
+ db: db, store: newFakeBackupStore(), pgDump: &fakePgDump{},
+ bucket: "b", prefix: "p",
+ aesKey: "not-hex-not-valid-please-fail",
+ now: time.Now, timeout: time.Minute, batchN: backupBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRunnerJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+// TestRunner_ProcessBackup_ManualKind_RefundsOnFailure — a kind='manual'
+// row that fails triggers the refund-quota call against the api.
+func TestRunner_ProcessBackup_ManualKind_RefundsOnFailure(t *testing.T) {
+ db, mock, err := sqlmock.New()
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+
+ backupID := "11111111-1111-1111-1111-111111111111"
+ resID := "22222222-2222-2222-2222-222222222222"
+ teamID := uuid.New()
+ enc := encryptForTest(t, "postgres://u:p@host/db")
+
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'pending'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT b.id::text`).
+ WithArgs(backupBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "tier_at_backup", "backup_kind",
+ "token", "connection_url", "resource_type", "team_id",
+ }).AddRow(backupID, resID, "pro", "manual", "tk", enc, "postgres", teamID))
+ mock.ExpectQuery(`UPDATE resource_backups\s+SET status = 'running'`).
+ WithArgs(backupID).
+ WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(backupID))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'failed'`).
+ WithArgs(backupID, sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ for i := 0; i < 5; i++ {
+ mock.ExpectQuery(`SELECT id::text, s3_key`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "s3_key"}))
+ }
+
+ refundCalls := 0
+ mu := sync.Mutex{}
+ apiSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ mu.Lock()
+ refundCalls++
+ mu.Unlock()
+ w.WriteHeader(http.StatusOK)
+ }))
+ defer apiSrv.Close()
+
+ w := &CustomerBackupRunnerWorker{
+ db: db, store: newFakeBackupStore(),
+ pgDump: &fakePgDump{err: errors.New("pg_dump down")},
+ bucket: "b", prefix: "p", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: backupBatchSize,
+ }
+ w.WithRefundClient(apiSrv.URL, "secret", &http.Client{Timeout: 2 * time.Second})
+
+ if err := w.Work(context.Background(), fakeRunnerJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+ mu.Lock()
+ defer mu.Unlock()
+ if refundCalls != 1 {
+ t.Errorf("refund POSTed %d times; want 1 (manual-kind failure must trigger refund)", refundCalls)
+ }
+}
+
+// TestRunner_RecoverStuckRows_DBError_LogsAndProceeds — exec error on the
+// recovery UPDATE is non-fatal.
+func TestRunner_RecoverStuckRows_DBError_LogsAndProceeds(t *testing.T) {
+ db, mock, err := sqlmock.New()
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'pending'`).
+ WillReturnError(errors.New("db gone"))
+ mock.ExpectQuery(`SELECT b.id::text`).
+ WithArgs(backupBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "tier_at_backup", "backup_kind",
+ "token", "connection_url", "resource_type", "team_id",
+ }))
+ for i := 0; i < 5; i++ {
+ mock.ExpectQuery(`SELECT id::text, s3_key`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "s3_key"}))
+ }
+
+ w := &CustomerBackupRunnerWorker{
+ db: db, store: newFakeBackupStore(), pgDump: &fakePgDump{},
+ bucket: "b", prefix: "p", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: backupBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRunnerJob()); err != nil {
+ t.Errorf("Work: stuck-row recovery error must be non-fatal: %v", err)
+ }
+}
+
+// TestRunner_RunRetentionSweep_DeletesAndUpdates — feed expired rows into
+// the per-tier sweep and assert the S3 deletes + DB s3_key=NULL updates.
+func TestRunner_RunRetentionSweep_DeletesAndUpdates(t *testing.T) {
+ db, mock, err := sqlmock.New()
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+
+ // No pending rows.
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'pending'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT b.id::text`).
+ WithArgs(backupBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "tier_at_backup", "backup_kind",
+ "token", "connection_url", "resource_type", "team_id",
+ }))
+
+ // Per-tier (5 tiers): first SELECT for tier "hobby" returns one expired
+ // row; remaining four tiers return empty.
+ mock.ExpectQuery(`SELECT id::text, s3_key`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "s3_key"}).
+ AddRow("99999999-9999-9999-9999-999999999999", "backups/tk/expired.dump.gz"))
+ // After SELECT for "hobby" the loop will iterate the one victim,
+ // calling DeleteObject (no DB) and then UPDATE resource_backups.
+ mock.ExpectExec(`UPDATE resource_backups\s+SET s3_key = NULL`).
+ WithArgs("99999999-9999-9999-9999-999999999999").
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ for i := 0; i < 4; i++ {
+ mock.ExpectQuery(`SELECT id::text, s3_key`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "s3_key"}))
+ }
+
+ store := newFakeBackupStore()
+ w := &CustomerBackupRunnerWorker{
+ db: db, store: store, pgDump: &fakePgDump{},
+ bucket: "b", prefix: "p", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: backupBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRunnerJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+ if len(store.deletes) != 1 {
+ t.Errorf("retention sweep should have deleted 1 expired object, got %d", len(store.deletes))
+ }
+}
+
+// TestRunner_RunRetentionSweep_UsesPlansRegistry — when plans is set, the
+// tier iteration order comes from registry.TierNames(), not the hardcoded
+// fallback. Asserted by counting the SELECT calls.
+func TestRunner_RunRetentionSweep_UsesPlansRegistry(t *testing.T) {
+ db, mock, err := sqlmock.New()
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'pending'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT b.id::text`).
+ WithArgs(backupBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "tier_at_backup", "backup_kind",
+ "token", "connection_url", "resource_type", "team_id",
+ }))
+
+ // Custom registry with 2 tiers — sweep should fire exactly 2 SELECTs.
+ reg := &fakeBackupPlanRegistry{
+ tiers: []string{"alpha", "beta"},
+ days: map[string]int{"alpha": 1, "beta": 2},
+ }
+ for i := 0; i < len(reg.tiers); i++ {
+ mock.ExpectQuery(`SELECT id::text, s3_key`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "s3_key"}))
+ }
+
+ w := &CustomerBackupRunnerWorker{
+ db: db, store: newFakeBackupStore(), pgDump: &fakePgDump{},
+ bucket: "b", prefix: "p", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: backupBatchSize,
+ plans: reg,
+ }
+ if err := w.Work(context.Background(), fakeRunnerJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+ if err := mock.ExpectationsWereMet(); err != nil {
+ t.Errorf("registry-driven sweep mismatch: %v", err)
+ }
+}
+
+// ────────────────────────────────────────────────────────────────────
+// customer_backup_scheduler.go — Kind
+// ────────────────────────────────────────────────────────────────────
+
+func TestCustomerBackupSchedulerArgs_Kind(t *testing.T) {
+ if got := (CustomerBackupSchedulerArgs{}).Kind(); got != "customer_backup_scheduler" {
+ t.Errorf("Kind() = %q", got)
+ }
+}
+
+// TestScheduler_AnonymousTier_DoesNotInsert — defensive: an anonymous row
+// in resource.tier slips the SQL filter (it shouldn't, but the cadence
+// switch also gates it). canonicalTier returns "anonymous"; the switch
+// has no case for it, so the row proceeds to the dedupe INSERT — which
+// is fine because the SQL filter excludes anonymous-tier rows in the
+// first place. This test pins that contract via the SELECT shape.
+func TestScheduler_AnonymousTier_DoesNotInsert(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+
+ // SELECT returns nothing — the SQL WHERE clause excludes anonymous.
+ mock.ExpectQuery(`SELECT r\.id::text`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "tier", "team_id"}))
+
+ w := NewCustomerBackupSchedulerWorker(db)
+ w.now = func() time.Time { return time.Date(2026, 5, 13, 0, 0, 0, 0, time.UTC) }
+ if err := w.Work(context.Background(), fakeSchedulerJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+// TestScheduler_HobbyMissingTeamID_Skips — defensive: a hobby-tier row
+// with NULL team_id is skipped (no panic-divide on the slot calc).
+func TestScheduler_HobbyMissingTeamID_Skips(t *testing.T) {
+ db, mock, err := sqlmock.New()
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+
+ resID := "fffffff0-1111-2222-3333-444444444444"
+ mock.ExpectQuery(`SELECT r.id::text`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "tier", "team_id"}).
+ AddRow(resID, "hobby", nil))
+ // No INSERT expected.
+
+ w := NewCustomerBackupSchedulerWorker(db)
+ w.now = func() time.Time { return time.Date(2026, 5, 13, 0, 0, 0, 0, time.UTC) }
+ if err := w.Work(context.Background(), fakeSchedulerJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+// TestScheduler_InsertError_LoggedNonFatal — an INSERT failure is logged
+// per-row and the sweep continues.
+func TestScheduler_InsertError_LoggedNonFatal(t *testing.T) {
+ db, mock, err := sqlmock.New()
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+
+ teamID := uuid.MustParse("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee")
+ resID := "fffffff0-1111-2222-3333-444444444444"
+
+ mock.ExpectQuery(`SELECT r.id::text`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "tier", "team_id"}).
+ AddRow(resID, "pro", teamID))
+ mock.ExpectExec(`INSERT INTO resource_backups`).
+ WithArgs(uuid.MustParse(resID), "pro").
+ WillReturnError(errors.New("db hiccup"))
+
+ w := NewCustomerBackupSchedulerWorker(db)
+ w.now = func() time.Time { return time.Date(2026, 5, 13, 14, 0, 0, 0, time.UTC) }
+ if err := w.Work(context.Background(), fakeSchedulerJob()); err != nil {
+ t.Errorf("Work: per-row insert error must be non-fatal: %v", err)
+ }
+}
+
+// TestScheduler_BadUUIDInRow_Skipped — a non-UUID id from the SELECT is
+// logged + skipped rather than crashing the sweep.
+func TestScheduler_BadUUIDInRow_Skipped(t *testing.T) {
+ db, mock, err := sqlmock.New()
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+
+ teamID := uuid.MustParse("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee")
+
+ mock.ExpectQuery(`SELECT r.id::text`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "tier", "team_id"}).
+ AddRow("not-a-uuid", "pro", teamID))
+ // No INSERT expected — bad UUID short-circuits the per-row body.
+
+ w := NewCustomerBackupSchedulerWorker(db)
+ w.now = func() time.Time { return time.Date(2026, 5, 13, 14, 0, 0, 0, time.UTC) }
+ if err := w.Work(context.Background(), fakeSchedulerJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+// ────────────────────────────────────────────────────────────────────
+// customer_restore_runner.go — Kind / ctor / Work edge branches
+// ────────────────────────────────────────────────────────────────────
+
+func TestCustomerRestoreRunnerArgs_Kind(t *testing.T) {
+ if got := (CustomerRestoreRunnerArgs{}).Kind(); got != "customer_restore_runner" {
+ t.Errorf("Kind() = %q", got)
+ }
+}
+
+func TestNewCustomerRestoreRunner_Defaults(t *testing.T) {
+ db, _, _ := sqlmock.New()
+ defer db.Close()
+ w := NewCustomerRestoreRunner(db, newFakeBackupStore(), "bucket", testAESKeyHex)
+ if w == nil || w.bucket != "bucket" || w.aesKey != testAESKeyHex {
+ t.Fatalf("ctor wiring wrong: %+v", w)
+ }
+ if w.timeout != restorePerRunTimeout {
+ t.Errorf("default timeout wrong")
+ }
+ if w.batchN != restoreBatchSize {
+ t.Errorf("default batch wrong")
+ }
+}
+
+// TestRestoreRunner_Work_SelectError_ReturnsError — SELECT-time DB outage
+// surfaces a Work error.
+func TestRestoreRunner_Work_SelectError_ReturnsError(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ mock.ExpectExec(`UPDATE resource_restores\s+SET status\s+= 'failed'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT rr\.id::text`).
+ WithArgs(restoreBatchSize).
+ WillReturnError(errors.New("db gone"))
+
+ w := &CustomerRestoreRunnerWorker{
+ db: db, store: newFakeBackupStore(), pgRestore: &fakePgRestore{},
+ bucket: "b", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: restoreBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRestoreJob()); err == nil {
+ t.Fatal("expected error, got nil")
+ }
+}
+
+// TestRestoreRunner_EmptyConnURL_Fails — connection_url NULL fails the
+// restore row immediately.
+func TestRestoreRunner_EmptyConnURL_Fails(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ restoreID := "rrrrrrr0-1111-2222-3333-444444444444"
+ resID := "22222222-2222-2222-2222-222222222222"
+ teamID := uuid.New()
+ s3Key := "backups/tk/abc.dump.gz"
+ store := newFakeBackupStore()
+ store.objects["instant-shared/"+s3Key] = gzipFor(t, []byte("p"))
+
+ mock.ExpectExec(`UPDATE resource_restores\s+SET status\s+= 'failed'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT rr\.id::text`).
+ WithArgs(restoreBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "backup_id", "s3_key", "sha256",
+ "connection_url", "resource_type", "token", "team_id",
+ }).AddRow(restoreID, resID, "bk", s3Key, nil, nil, "postgres", "tk", teamID))
+ mock.ExpectQuery(`UPDATE resource_restores\s+SET status = 'running'`).
+ WithArgs(restoreID).
+ WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(restoreID))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`UPDATE resource_restores\s+SET status = 'failed'`).
+ WithArgs(restoreID, sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+
+ w := &CustomerRestoreRunnerWorker{
+ db: db, store: store, pgRestore: &fakePgRestore{},
+ bucket: "instant-shared", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: restoreBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRestoreJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+// TestRestoreRunner_BadAESKey_Fails — invalid AES key short-circuits with
+// markRestoreFailed.
+func TestRestoreRunner_BadAESKey_Fails(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ restoreID := "rrrrrrr0-1111-2222-3333-444444444444"
+ resID := "22222222-2222-2222-2222-222222222222"
+ teamID := uuid.New()
+ s3Key := "backups/tk/abc.dump.gz"
+ store := newFakeBackupStore()
+ store.objects["instant-shared/"+s3Key] = gzipFor(t, []byte("p"))
+ enc := encryptForTest(t, "postgres://u:p@host/db")
+
+ mock.ExpectExec(`UPDATE resource_restores\s+SET status\s+= 'failed'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT rr\.id::text`).
+ WithArgs(restoreBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "backup_id", "s3_key", "sha256",
+ "connection_url", "resource_type", "token", "team_id",
+ }).AddRow(restoreID, resID, "bk", s3Key, nil, enc, "postgres", "tk", teamID))
+ mock.ExpectQuery(`UPDATE resource_restores\s+SET status = 'running'`).
+ WithArgs(restoreID).
+ WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(restoreID))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`UPDATE resource_restores\s+SET status = 'failed'`).
+ WithArgs(restoreID, sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+
+ w := &CustomerRestoreRunnerWorker{
+ db: db, store: store, pgRestore: &fakePgRestore{},
+ bucket: "instant-shared",
+ aesKey: "not-hex-ok",
+ now: time.Now, timeout: time.Minute, batchN: restoreBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRestoreJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+// TestRestoreRunner_S3DownloadError_Fails — store.Download returns an
+// error → markRestoreFailed.
+func TestRestoreRunner_S3DownloadError_Fails(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ restoreID := "rrrrrrr0-1111-2222-3333-444444444444"
+ resID := "22222222-2222-2222-2222-222222222222"
+ teamID := uuid.New()
+ enc := encryptForTest(t, "postgres://u:p@host/db")
+
+ mock.ExpectExec(`UPDATE resource_restores\s+SET status\s+= 'failed'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT rr\.id::text`).
+ WithArgs(restoreBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "backup_id", "s3_key", "sha256",
+ "connection_url", "resource_type", "token", "team_id",
+ }).AddRow(restoreID, resID, "bk", "backups/tk/missing.dump.gz", nil, enc, "postgres", "tk", teamID))
+ mock.ExpectQuery(`UPDATE resource_restores\s+SET status = 'running'`).
+ WithArgs(restoreID).
+ WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(restoreID))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`UPDATE resource_restores\s+SET status = 'failed'`).
+ WithArgs(restoreID, sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+
+ w := &CustomerRestoreRunnerWorker{
+ db: db, store: newFakeBackupStore(), // object NOT seeded → Download errors
+ pgRestore: &fakePgRestore{},
+ bucket: "instant-shared", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: restoreBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRestoreJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+// TestRestoreRunner_GzipHeaderInvalid_Fails — the S3 object is NOT gzip
+// (it's plain text). The gzip.NewReader header check fails before pg_restore
+// is invoked.
+func TestRestoreRunner_GzipHeaderInvalid_Fails(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ restoreID := "rrrrrrr0-1111-2222-3333-444444444444"
+ resID := "22222222-2222-2222-2222-222222222222"
+ teamID := uuid.New()
+ enc := encryptForTest(t, "postgres://u:p@host/db")
+ s3Key := "backups/tk/garbage.dump.gz"
+ store := newFakeBackupStore()
+ store.objects["instant-shared/"+s3Key] = []byte("NOT-A-GZIP-STREAM-AT-ALL")
+
+ mock.ExpectExec(`UPDATE resource_restores\s+SET status\s+= 'failed'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT rr\.id::text`).
+ WithArgs(restoreBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "backup_id", "s3_key", "sha256",
+ "connection_url", "resource_type", "token", "team_id",
+ }).AddRow(restoreID, resID, "bk", s3Key, nil, enc, "postgres", "tk", teamID))
+ mock.ExpectQuery(`UPDATE resource_restores\s+SET status = 'running'`).
+ WithArgs(restoreID).
+ WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(restoreID))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`UPDATE resource_restores\s+SET status = 'failed'`).
+ WithArgs(restoreID, sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+
+ pgr := &fakePgRestore{}
+ w := &CustomerRestoreRunnerWorker{
+ db: db, store: store, pgRestore: pgr,
+ bucket: "instant-shared", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: restoreBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRestoreJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+ // pg_restore must NOT have run on an invalid gzip header.
+ if pgr.gotConn != "" {
+ t.Errorf("pg_restore invoked on bad gzip header — should have been gated")
+ }
+}
+
+// TestRestoreRunner_ClaimRace_Skips — another worker already grabbed the
+// row; our UPDATE returns 0 rows and we move on silently.
+func TestRestoreRunner_ClaimRace_Skips(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ restoreID := "rrrrrrr0-1111-2222-3333-444444444444"
+ resID := "22222222-2222-2222-2222-222222222222"
+ teamID := uuid.New()
+ enc := encryptForTest(t, "postgres://u:p@host/db")
+
+ mock.ExpectExec(`UPDATE resource_restores\s+SET status\s+= 'failed'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT rr\.id::text`).
+ WithArgs(restoreBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "backup_id", "s3_key", "sha256",
+ "connection_url", "resource_type", "token", "team_id",
+ }).AddRow(restoreID, resID, "bk", "backups/tk/abc.dump.gz", nil, enc, "postgres", "tk", teamID))
+ // 0-row claim — competing worker.
+ mock.ExpectQuery(`UPDATE resource_restores\s+SET status = 'running'`).
+ WithArgs(restoreID).
+ WillReturnRows(sqlmock.NewRows([]string{"id"}))
+
+ w := &CustomerRestoreRunnerWorker{
+ db: db, store: newFakeBackupStore(), pgRestore: &fakePgRestore{},
+ bucket: "instant-shared", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: restoreBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRestoreJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+// TestRestoreRunner_RecoverStuckRestores_HitsRow — exec returns
+// RowsAffected=N so the WARN log line fires; assert no Work-level error.
+func TestRestoreRunner_RecoverStuckRestores_HitsRow(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+
+ mock.ExpectExec(`UPDATE resource_restores\s+SET status\s+= 'failed'`).
+ WillReturnResult(sqlmock.NewResult(0, 3))
+ mock.ExpectQuery(`SELECT rr\.id::text`).
+ WithArgs(restoreBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "backup_id", "s3_key", "sha256",
+ "connection_url", "resource_type", "token", "team_id",
+ }))
+
+ w := &CustomerRestoreRunnerWorker{
+ db: db, store: newFakeBackupStore(), pgRestore: &fakePgRestore{},
+ bucket: "instant-shared", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: restoreBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRestoreJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+// TestRestoreRunner_RecoverStuckRestores_DBError_LogsAndProceeds — UPDATE
+// fails; the sweep proceeds without bubbling.
+func TestRestoreRunner_RecoverStuckRestores_DBError_LogsAndProceeds(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ mock.ExpectExec(`UPDATE resource_restores\s+SET status\s+= 'failed'`).
+ WillReturnError(errors.New("db hiccup"))
+ mock.ExpectQuery(`SELECT rr\.id::text`).
+ WithArgs(restoreBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "backup_id", "s3_key", "sha256",
+ "connection_url", "resource_type", "token", "team_id",
+ }))
+
+ w := &CustomerRestoreRunnerWorker{
+ db: db, store: newFakeBackupStore(), pgRestore: &fakePgRestore{},
+ bucket: "instant-shared", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: restoreBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRestoreJob()); err != nil {
+ t.Errorf("recoverStuckRestores DB error must be non-fatal: %v", err)
+ }
+}
+
+// (Removed TestRestoreRunner_ContextCancelledMidBatch: identical reasoning
+// to the runner sibling — pre-cancel kills the SELECT before the per-row
+// ctx.Done() check is reachable.)
+
+// ────────────────────────────────────────────────────────────────────
+// platform_db_backup.go — Kind / ctors / Work edge branches /
+// joinPlatformBackupPrefix / defaultPgDumpExec via a fake binary
+// ────────────────────────────────────────────────────────────────────
+
+func TestPlatformDBBackupArgs_Kind(t *testing.T) {
+ if got := (PlatformDBBackupArgs{}).Kind(); got != "platform_db_backup" {
+ t.Errorf("Kind() = %q", got)
+ }
+}
+
+// TestNewPlatformDBBackupWorker_DefaultsApplied — nil Dumper / Now fall
+// back to the package defaults.
+func TestNewPlatformDBBackupWorker_DefaultsApplied(t *testing.T) {
+ w := NewPlatformDBBackupWorker(PlatformDBBackupConfig{
+ DatabaseURL: "postgres://x@y/z",
+ Bucket: "b",
+ OuterPrefix: "",
+ InnerPrefix: "platform-backups/",
+ Dumper: nil, // ← exercise the default
+ Now: nil, // ← exercise the default
+ })
+ if w == nil {
+ t.Fatal("nil worker")
+ }
+ if w.now == nil {
+ t.Error("now not defaulted")
+ }
+ if w.dumper == nil {
+ t.Error("dumper not defaulted")
+ }
+ if w.keyPrefix != "platform-backups/" {
+ t.Errorf("keyPrefix = %q", w.keyPrefix)
+ }
+}
+
+// TestJoinPlatformBackupPrefix_AllShapes — covers every branch of the
+// prefix join: empty outer, empty inner, both, trailing slashes.
+func TestJoinPlatformBackupPrefix_AllShapes(t *testing.T) {
+ cases := []struct {
+ outer, inner, want string
+ }{
+ {"", "", "platform-backups/"}, // defensive default
+ {"", "platform-backups/", "platform-backups/"},
+ {"backups/", "", "backups/"},
+ {"backups", "platform-backups", "backups/platform-backups/"},
+ {"/backups/", "/platform-backups/", "backups/platform-backups/"},
+ }
+ for _, c := range cases {
+ got := joinPlatformBackupPrefix(c.outer, c.inner)
+ if got != c.want {
+ t.Errorf("joinPlatformBackupPrefix(%q,%q) = %q; want %q", c.outer, c.inner, got, c.want)
+ }
+ }
+}
+
+// TestPlatformDBBackup_NoBucket_Skips — bucket-empty disabled-mode branch.
+func TestPlatformDBBackup_NoBucket_Skips(t *testing.T) {
+ db, _, _ := sqlmock.New(sqlmock.MonitorPingsOption(false))
+ defer db.Close()
+ w := NewPlatformDBBackupWorker(PlatformDBBackupConfig{
+ DB: db,
+ DatabaseURL: "postgres://x",
+ S3: newFakeS3(),
+ Bucket: "", // ← empty
+ InnerPrefix: "platform-backups/",
+ Now: fixedClock(time.Date(2026, 5, 13, 2, 0, 0, 0, time.UTC)),
+ })
+ if err := w.Work(context.Background(), fakePlatformBackupJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+// TestPlatformDBBackup_NoDatabaseURL_Errors — empty DSN is a defensive
+// error, not silent skip.
+func TestPlatformDBBackup_NoDatabaseURL_Errors(t *testing.T) {
+ db, _, _ := sqlmock.New(sqlmock.MonitorPingsOption(false))
+ defer db.Close()
+ w := NewPlatformDBBackupWorker(PlatformDBBackupConfig{
+ DB: db,
+ DatabaseURL: "",
+ S3: newFakeS3(),
+ Bucket: "b",
+ InnerPrefix: "platform-backups/",
+ Now: fixedClock(time.Date(2026, 5, 13, 0, 0, 0, 0, time.UTC)),
+ })
+ if err := w.Work(context.Background(), fakePlatformBackupJob()); err == nil {
+ t.Fatal("expected error for empty DATABASE_URL")
+ }
+}
+
+// TestPlatformDBBackup_UploadError_FailsLoud — pg_dump succeeds but the
+// S3 Upload fails. Work returns an error AND a failed audit row fires.
+func TestPlatformDBBackup_UploadError_FailsLoud(t *testing.T) {
+ db, mock, err := sqlmock.New(
+ sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp),
+ sqlmock.MonitorPingsOption(false),
+ )
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+
+ expectAdvisoryLockAcquired(mock)
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs("system", "platform_backup.started", sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs("system", "platform_backup.failed", sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ expectAdvisoryUnlock(mock)
+
+ dumper := &fakePgDumper{payload: []byte("data")}
+ s3 := newFakeS3()
+ s3.uploadErr = errors.New("s3 down")
+ now := time.Date(2026, 5, 13, 2, 0, 0, 0, time.UTC)
+ w := newTestWorker(t, mock, db, dumper, s3, now)
+ err = w.Work(context.Background(), fakePlatformBackupJob())
+ if err == nil {
+ t.Fatal("expected error on upload failure")
+ }
+ if !strings.Contains(err.Error(), "s3 upload") {
+ t.Errorf("error wrap missing s3 upload: %v", err)
+ }
+}
+
+// TestPlatformDBBackup_ListError_StillReturnsNil — retention sweep List
+// failure is non-fatal: the upload itself succeeded.
+func TestPlatformDBBackup_ListError_StillReturnsNil(t *testing.T) {
+ db, mock, err := sqlmock.New(
+ sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp),
+ sqlmock.MonitorPingsOption(false),
+ )
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+
+ expectAdvisoryLockAcquired(mock)
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs("system", "platform_backup.started", sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs("system", "platform_backup.succeeded", sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ expectAdvisoryUnlock(mock)
+
+ dumper := &fakePgDumper{payload: []byte("ok")}
+ s3 := newFakeS3()
+ s3.listErr = errors.New("list 500")
+ now := time.Date(2026, 5, 13, 2, 0, 0, 0, time.UTC)
+ w := newTestWorker(t, mock, db, dumper, s3, now)
+ if err := w.Work(context.Background(), fakePlatformBackupJob()); err != nil {
+ t.Errorf("Work: list error must be non-fatal: %v", err)
+ }
+}
+
+// TestPlatformDBBackup_WriteAudit_NilDB_Skips — the writeAudit helper
+// short-circuits on nil DB. We exercise this by constructing a worker
+// with no DB and calling writeAudit directly.
+func TestPlatformDBBackup_WriteAudit_NilDB_Skips(t *testing.T) {
+ w := NewPlatformDBBackupWorker(PlatformDBBackupConfig{
+ DB: nil,
+ DatabaseURL: "postgres://x",
+ S3: newFakeS3(),
+ Bucket: "b",
+ InnerPrefix: "platform-backups/",
+ Now: fixedClock(time.Date(2026, 5, 13, 0, 0, 0, 0, time.UTC)),
+ })
+ w.writeAudit(context.Background(), "kind", "summary", map[string]any{"k": "v"})
+ // No panic == pass.
+}
+
+// TestDurationSeconds_Rounds — pure helper.
+func TestDurationSeconds_Rounds(t *testing.T) {
+ if got := durationSeconds(1234 * time.Millisecond); got != 1.2 {
+ t.Errorf("durationSeconds(1.234s) = %v; want 1.2", got)
+ }
+ if got := durationSeconds(0); got != 0 {
+ t.Errorf("durationSeconds(0) = %v; want 0", got)
+ }
+}
+
+// TestDefaultPgDumpExec_BadBinary — invalid PG_DUMP_BIN → start error.
+func TestDefaultPgDumpExec_BadBinary(t *testing.T) {
+ t.Setenv("PG_DUMP_BIN", "/nonexistent/path/to/pg_dump_xyz")
+ d := defaultPgDumpExec{}
+ _, err := d.Dump(context.Background(), "postgres://x", io.Discard)
+ if err == nil {
+ t.Fatal("expected error from missing pg_dump binary")
+ }
+}
+
+// TestDefaultPgDumpExec_FakeBinarySuccess — point PG_DUMP_BIN at a tiny
+// shell script that emits N bytes and exits 0. Asserts the bytes round-
+// trip into the writer and the byte count is reported. Skipped on Windows
+// (no /bin/sh).
+func TestDefaultPgDumpExec_FakeBinarySuccess(t *testing.T) {
+ if runtime.GOOS == "windows" {
+ t.Skip("no /bin/sh on windows")
+ }
+ dir := t.TempDir()
+ bin := filepath.Join(dir, "fake_pg_dump")
+ script := "#!/bin/sh\nprintf 'HELLO'\n"
+ if err := os.WriteFile(bin, []byte(script), 0o755); err != nil {
+ t.Fatalf("write fake bin: %v", err)
+ }
+ t.Setenv("PG_DUMP_BIN", bin)
+ d := defaultPgDumpExec{}
+ var sink strings.Builder
+ n, err := d.Dump(context.Background(), "postgres://x", &sink)
+ if err != nil {
+ t.Fatalf("Dump: %v", err)
+ }
+ if n != 5 {
+ t.Errorf("n = %d; want 5", n)
+ }
+ if sink.String() != "HELLO" {
+ t.Errorf("sink = %q; want HELLO", sink.String())
+ }
+}
+
+// TestDefaultPgDumpExec_FakeBinaryNonZeroExit — fake binary exits 1; the
+// stderr is captured into the error message.
+func TestDefaultPgDumpExec_FakeBinaryNonZeroExit(t *testing.T) {
+ if runtime.GOOS == "windows" {
+ t.Skip("no /bin/sh on windows")
+ }
+ dir := t.TempDir()
+ bin := filepath.Join(dir, "fake_pg_dump_fail")
+ script := "#!/bin/sh\necho 'pg_dump: stderr text' >&2\nexit 1\n"
+ if err := os.WriteFile(bin, []byte(script), 0o755); err != nil {
+ t.Fatalf("write: %v", err)
+ }
+ t.Setenv("PG_DUMP_BIN", bin)
+ d := defaultPgDumpExec{}
+ var sink strings.Builder
+ _, err := d.Dump(context.Background(), "postgres://x", &sink)
+ if err == nil {
+ t.Fatal("expected non-zero exit error")
+ }
+ if !strings.Contains(err.Error(), "pg_dump exit") {
+ t.Errorf("error not wrapped: %v", err)
+ }
+}
+
+// TestDefaultPgDumpExec_StderrTruncates — stderr > 256 chars is trimmed
+// in the wrapped error message.
+func TestDefaultPgDumpExec_StderrTruncates(t *testing.T) {
+ if runtime.GOOS == "windows" {
+ t.Skip("no /bin/sh on windows")
+ }
+ dir := t.TempDir()
+ bin := filepath.Join(dir, "noisy_pg_dump")
+ // Write 1000 bytes to stderr then exit 1.
+ script := fmt.Sprintf("#!/bin/sh\nfor i in $(seq 1 100); do printf 'XXXXXXXXXX' >&2; done\nexit 1\n")
+ if err := os.WriteFile(bin, []byte(script), 0o755); err != nil {
+ t.Fatalf("write: %v", err)
+ }
+ t.Setenv("PG_DUMP_BIN", bin)
+ d := defaultPgDumpExec{}
+ var sink strings.Builder
+ _, err := d.Dump(context.Background(), "postgres://x", &sink)
+ if err == nil {
+ t.Fatal("expected error")
+ }
+ if !strings.Contains(err.Error(), "...(truncated)") {
+ t.Errorf("expected stderr truncation marker in error: %v", err)
+ }
+}
+
+// ────────────────────────────────────────────────────────────────────
+// finalizeDigest — null hash path
+// ────────────────────────────────────────────────────────────────────
+
+// ────────────────────────────────────────────────────────────────────
+// realPgDumpRunner / realPgRestoreRunner — exercised via PATH tricks.
+// ────────────────────────────────────────────────────────────────────
+
+// withFakeBinaryOnPath puts a tiny shell script named `name` on PATH ahead
+// of any system binary, returning the dir so callers can also reach
+// it for assertion. Each test gets its own temp dir.
+func withFakeBinaryOnPath(t *testing.T, name, body string) string {
+ t.Helper()
+ if runtime.GOOS == "windows" {
+ t.Skip("no /bin/sh on windows")
+ }
+ dir := t.TempDir()
+ bin := filepath.Join(dir, name)
+ if err := os.WriteFile(bin, []byte(body), 0o755); err != nil {
+ t.Fatalf("write %s: %v", name, err)
+ }
+ t.Setenv("PATH", dir+":"+os.Getenv("PATH"))
+ return dir
+}
+
+// TestRealPgDumpRunner_Run_Success — point a shell-script `pg_dump` at the
+// runner; assert the bytes round-trip into the writer.
+func TestRealPgDumpRunner_Run_Success(t *testing.T) {
+ _ = withFakeBinaryOnPath(t, "pg_dump", "#!/bin/sh\nprintf 'PGDUMP-OUT'\n")
+ r := realPgDumpRunner{}
+ var sink strings.Builder
+ if err := r.Run(context.Background(), "postgres://x", &sink); err != nil {
+ t.Fatalf("Run: %v", err)
+ }
+ if sink.String() != "PGDUMP-OUT" {
+ t.Errorf("sink = %q", sink.String())
+ }
+}
+
+// TestRealPgDumpRunner_Run_NonZeroExit — fake `pg_dump` exits 1 with a
+// stderr message; the wrapper captures it into the returned error.
+func TestRealPgDumpRunner_Run_NonZeroExit(t *testing.T) {
+ _ = withFakeBinaryOnPath(t, "pg_dump", "#!/bin/sh\necho 'pg_dump: server unavailable' >&2\nexit 1\n")
+ r := realPgDumpRunner{}
+ err := r.Run(context.Background(), "postgres://x", io.Discard)
+ if err == nil {
+ t.Fatal("expected error")
+ }
+ if !strings.Contains(err.Error(), "pg_dump") {
+ t.Errorf("error not wrapped: %v", err)
+ }
+}
+
+// TestRealPgRestoreRunner_Run_Success — same trick for pg_restore. The
+// reader is drained into the subprocess's stdin; here we just verify the
+// process runs to completion (the stub doesn't actually consume stdin).
+func TestRealPgRestoreRunner_Run_Success(t *testing.T) {
+ _ = withFakeBinaryOnPath(t, "pg_restore", "#!/bin/sh\ncat >/dev/null\nexit 0\n")
+ r := realPgRestoreRunner{}
+ if err := r.Run(context.Background(), "postgres://x", strings.NewReader("ignored")); err != nil {
+ t.Fatalf("Run: %v", err)
+ }
+}
+
+// TestRealPgRestoreRunner_Run_NonZeroExit — fake `pg_restore` errors out.
+func TestRealPgRestoreRunner_Run_NonZeroExit(t *testing.T) {
+ _ = withFakeBinaryOnPath(t, "pg_restore", "#!/bin/sh\necho 'pg_restore: bad dump' >&2\nexit 1\n")
+ r := realPgRestoreRunner{}
+ err := r.Run(context.Background(), "postgres://x", strings.NewReader(""))
+ if err == nil {
+ t.Fatal("expected error")
+ }
+ if !strings.Contains(err.Error(), "pg_restore") {
+ t.Errorf("error not wrapped: %v", err)
+ }
+}
+
+// ────────────────────────────────────────────────────────────────────
+// customer_backup_runner.go — remaining branches
+// ────────────────────────────────────────────────────────────────────
+
+// failingStoreOnDelete is a BackupObjectStore that succeeds on Upload but
+// errors on DeleteObject — used to exercise the retention-sweep s3-delete
+// error path AND the cleanup-after-dump-failure error log path.
+type failingStoreOnDelete struct {
+ *fakeBackupStore
+ delErr error
+}
+
+func (f *failingStoreOnDelete) DeleteObject(ctx context.Context, bucket, key string) error {
+ if f.delErr != nil {
+ return f.delErr
+ }
+ return f.fakeBackupStore.DeleteObject(ctx, bucket, key)
+}
+
+// TestRunner_RetentionSweep_S3DeleteError_LogsContinues — when the
+// retention DeleteObject fails for one victim, the loop continues for
+// other victims AND the DB UPDATE for that victim is skipped. Asserted
+// by sqlmock — no UPDATE expectation primed for the failing victim.
+func TestRunner_RetentionSweep_S3DeleteError_LogsContinues(t *testing.T) {
+ db, mock, err := sqlmock.New()
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'pending'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT b.id::text`).
+ WithArgs(backupBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "tier_at_backup", "backup_kind",
+ "token", "connection_url", "resource_type", "team_id",
+ }))
+
+ // First-tier SELECT returns one victim; store.DeleteObject errors → no
+ // matching UPDATE primed.
+ mock.ExpectQuery(`SELECT id::text, s3_key`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "s3_key"}).
+ AddRow("99999999-9999-9999-9999-999999999999", "backups/tk/expired.dump.gz"))
+ for i := 0; i < 4; i++ {
+ mock.ExpectQuery(`SELECT id::text, s3_key`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "s3_key"}))
+ }
+
+ store := &failingStoreOnDelete{
+ fakeBackupStore: newFakeBackupStore(),
+ delErr: errors.New("s3 503"),
+ }
+ w := &CustomerBackupRunnerWorker{
+ db: db, store: store, pgDump: &fakePgDump{},
+ bucket: "b", prefix: "p", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: backupBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRunnerJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+// TestRunner_RetentionSweep_DBUpdateError_LogsContinues — DeleteObject
+// succeeds but the follow-up s3_key=NULL UPDATE fails for one victim.
+func TestRunner_RetentionSweep_DBUpdateError_LogsContinues(t *testing.T) {
+ db, mock, err := sqlmock.New()
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'pending'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT b.id::text`).
+ WithArgs(backupBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "tier_at_backup", "backup_kind",
+ "token", "connection_url", "resource_type", "team_id",
+ }))
+
+ mock.ExpectQuery(`SELECT id::text, s3_key`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "s3_key"}).
+ AddRow("11111111-1111-1111-1111-111111111111", "backups/tk/old.dump.gz"))
+ mock.ExpectExec(`UPDATE resource_backups\s+SET s3_key = NULL`).
+ WithArgs("11111111-1111-1111-1111-111111111111").
+ WillReturnError(errors.New("db hiccup"))
+ for i := 0; i < 4; i++ {
+ mock.ExpectQuery(`SELECT id::text, s3_key`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "s3_key"}))
+ }
+
+ w := &CustomerBackupRunnerWorker{
+ db: db, store: newFakeBackupStore(), pgDump: &fakePgDump{},
+ bucket: "b", prefix: "p", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: backupBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRunnerJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+// TestRunner_RetentionSweep_QueryError_LogsContinues — first-tier SELECT
+// errors; the sweep continues to the remaining 4 tiers.
+func TestRunner_RetentionSweep_QueryError_LogsContinues(t *testing.T) {
+ db, mock, err := sqlmock.New()
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'pending'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT b.id::text`).
+ WithArgs(backupBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "tier_at_backup", "backup_kind",
+ "token", "connection_url", "resource_type", "team_id",
+ }))
+
+ mock.ExpectQuery(`SELECT id::text, s3_key`).WillReturnError(errors.New("tier query 500"))
+ for i := 0; i < 4; i++ {
+ mock.ExpectQuery(`SELECT id::text, s3_key`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "s3_key"}))
+ }
+
+ w := &CustomerBackupRunnerWorker{
+ db: db, store: newFakeBackupStore(), pgDump: &fakePgDump{},
+ bucket: "b", prefix: "p", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: backupBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRunnerJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+// TestRunner_ClaimDBError_LogsContinues — the claim UPDATE errors (non-
+// sql.ErrNoRows). The row is skipped, the rest of the batch proceeds.
+func TestRunner_ClaimDBError_LogsContinues(t *testing.T) {
+ db, mock, err := sqlmock.New()
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+
+ backupID := "11111111-1111-1111-1111-111111111111"
+ resID := "22222222-2222-2222-2222-222222222222"
+ teamID := uuid.New()
+ enc := encryptForTest(t, "postgres://u:p@host/db")
+
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'pending'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT b.id::text`).
+ WithArgs(backupBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "tier_at_backup", "backup_kind",
+ "token", "connection_url", "resource_type", "team_id",
+ }).AddRow(backupID, resID, "pro", "scheduled", "tk", enc, "postgres", teamID))
+ // Claim UPDATE fails with a non-ErrNoRows error.
+ mock.ExpectQuery(`UPDATE resource_backups\s+SET status = 'running'`).
+ WithArgs(backupID).
+ WillReturnError(errors.New("claim down"))
+ for i := 0; i < 5; i++ {
+ mock.ExpectQuery(`SELECT id::text, s3_key`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "s3_key"}))
+ }
+
+ w := &CustomerBackupRunnerWorker{
+ db: db, store: newFakeBackupStore(), pgDump: &fakePgDump{},
+ bucket: "b", prefix: "p", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: backupBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRunnerJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+// TestRunner_FinalizeUpdateError_LogsAndReturns — pg_dump + upload
+// succeed, but the finalize UPDATE errors. The slog.Error fires and
+// processBackup returns false (no panic).
+func TestRunner_FinalizeUpdateError_LogsAndReturns(t *testing.T) {
+ db, mock, err := sqlmock.New()
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+
+ backupID := "11111111-1111-1111-1111-111111111111"
+ resID := "22222222-2222-2222-2222-222222222222"
+ teamID := uuid.New()
+ enc := encryptForTest(t, "postgres://u:p@host/db")
+
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'pending'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT b.id::text`).
+ WithArgs(backupBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "tier_at_backup", "backup_kind",
+ "token", "connection_url", "resource_type", "team_id",
+ }).AddRow(backupID, resID, "pro", "scheduled", "tk", enc, "postgres", teamID))
+ mock.ExpectQuery(`UPDATE resource_backups\s+SET status = 'running'`).
+ WithArgs(backupID).
+ WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(backupID))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ // Finalize UPDATE returns an error.
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'ok'`).
+ WithArgs(backupID, sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnError(errors.New("finalize down"))
+ for i := 0; i < 5; i++ {
+ mock.ExpectQuery(`SELECT id::text, s3_key`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "s3_key"}))
+ }
+
+ w := &CustomerBackupRunnerWorker{
+ db: db, store: newFakeBackupStore(),
+ pgDump: &fakePgDump{payload: []byte("ok")},
+ bucket: "b", prefix: "p", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: backupBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRunnerJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+// TestRunner_MarkFailed_DBError_StillContinues — markFailed's UPDATE
+// errors; the audit row still attempts and the sweep proceeds.
+func TestRunner_MarkFailed_DBError_StillContinues(t *testing.T) {
+ db, mock, err := sqlmock.New()
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+
+ backupID := "11111111-1111-1111-1111-111111111111"
+ resID := "22222222-2222-2222-2222-222222222222"
+ teamID := uuid.New()
+ enc := encryptForTest(t, "postgres://u:p@host/db")
+
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'pending'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT b.id::text`).
+ WithArgs(backupBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "tier_at_backup", "backup_kind",
+ "token", "connection_url", "resource_type", "team_id",
+ }).AddRow(backupID, resID, "pro", "scheduled", "tk", enc, "postgres", teamID))
+ mock.ExpectQuery(`UPDATE resource_backups\s+SET status = 'running'`).
+ WithArgs(backupID).
+ WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(backupID))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'failed'`).
+ WithArgs(backupID, sqlmock.AnyArg()).
+ WillReturnError(errors.New("mark failed db err"))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ for i := 0; i < 5; i++ {
+ mock.ExpectQuery(`SELECT id::text, s3_key`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "s3_key"}))
+ }
+
+ w := &CustomerBackupRunnerWorker{
+ db: db, store: newFakeBackupStore(),
+ pgDump: &fakePgDump{err: errors.New("pg_dump down")},
+ bucket: "b", prefix: "p", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: backupBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRunnerJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+// TestRunner_WriteAudit_DBError_LogsAndContinues — audit insert errors.
+// We trigger this on the FIRST audit row (the 'started' row) by primed
+// returning an exec error there; the rest of the row processing still
+// completes (Upload succeeds, finalize updates).
+func TestRunner_WriteAudit_DBError_LogsAndContinues(t *testing.T) {
+ db, mock, err := sqlmock.New()
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+
+ backupID := "11111111-1111-1111-1111-111111111111"
+ resID := "22222222-2222-2222-2222-222222222222"
+ teamID := uuid.New()
+ enc := encryptForTest(t, "postgres://u:p@host/db")
+
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'pending'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT b.id::text`).
+ WithArgs(backupBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "tier_at_backup", "backup_kind",
+ "token", "connection_url", "resource_type", "team_id",
+ }).AddRow(backupID, resID, "pro", "scheduled", "tk", enc, "postgres", teamID))
+ mock.ExpectQuery(`UPDATE resource_backups\s+SET status = 'running'`).
+ WithArgs(backupID).
+ WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(backupID))
+ // started audit INSERT errors — runner continues.
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnError(errors.New("audit insert down"))
+ // Finalize UPDATE still happens because the backup itself proceeded.
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'ok'`).
+ WithArgs(backupID, sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ for i := 0; i < 5; i++ {
+ mock.ExpectQuery(`SELECT id::text, s3_key`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "s3_key"}))
+ }
+
+ w := &CustomerBackupRunnerWorker{
+ db: db, store: newFakeBackupStore(),
+ pgDump: &fakePgDump{payload: []byte("ok")},
+ bucket: "b", prefix: "p", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: backupBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRunnerJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+// TestRunner_ProcessBackup_UploadFails_MarksFailed — pg_dump emits bytes
+// but the store's Upload returns an error. The row goes to 'failed' and
+// the cleanup-delete path is exercised.
+func TestRunner_ProcessBackup_UploadFails_MarksFailed(t *testing.T) {
+ db, mock, err := sqlmock.New()
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+
+ backupID := "11111111-1111-1111-1111-111111111111"
+ resID := "22222222-2222-2222-2222-222222222222"
+ teamID := uuid.New()
+ enc := encryptForTest(t, "postgres://u:p@host/db")
+
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'pending'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT b.id::text`).
+ WithArgs(backupBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "tier_at_backup", "backup_kind",
+ "token", "connection_url", "resource_type", "team_id",
+ }).AddRow(backupID, resID, "pro", "scheduled", "tk", enc, "postgres", teamID))
+ mock.ExpectQuery(`UPDATE resource_backups\s+SET status = 'running'`).
+ WithArgs(backupID).
+ WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(backupID))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'failed'`).
+ WithArgs(backupID, sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ for i := 0; i < 5; i++ {
+ mock.ExpectQuery(`SELECT id::text, s3_key`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "s3_key"}))
+ }
+
+ store := newFakeBackupStore()
+ store.uploadFn = func(_ context.Context, _, _ string, r io.Reader) (int64, error) {
+ // Drain so the producer side isn't deadlocked.
+ _, _ = io.Copy(io.Discard, r)
+ return 0, errors.New("upload 503")
+ }
+ w := &CustomerBackupRunnerWorker{
+ db: db, store: store,
+ pgDump: &fakePgDump{payload: []byte("ok")},
+ bucket: "b", prefix: "p", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: backupBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRunnerJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+// TestRunner_PgDumpFails_CleanupDeleteAlsoFails — pg_dump errors + the
+// cleanup store.DeleteObject also errors. Both error logs fire; Work
+// returns nil (failures are per-row).
+func TestRunner_PgDumpFails_CleanupDeleteAlsoFails(t *testing.T) {
+ db, mock, err := sqlmock.New()
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+
+ backupID := "11111111-1111-1111-1111-111111111111"
+ resID := "22222222-2222-2222-2222-222222222222"
+ teamID := uuid.New()
+ enc := encryptForTest(t, "postgres://u:p@host/db")
+
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'pending'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT b.id::text`).
+ WithArgs(backupBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "tier_at_backup", "backup_kind",
+ "token", "connection_url", "resource_type", "team_id",
+ }).AddRow(backupID, resID, "pro", "scheduled", "tk", enc, "postgres", teamID))
+ mock.ExpectQuery(`UPDATE resource_backups\s+SET status = 'running'`).
+ WithArgs(backupID).
+ WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(backupID))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'failed'`).
+ WithArgs(backupID, sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ for i := 0; i < 5; i++ {
+ mock.ExpectQuery(`SELECT id::text, s3_key`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "s3_key"}))
+ }
+
+ store := &failingStoreOnDelete{
+ fakeBackupStore: newFakeBackupStore(),
+ delErr: errors.New("s3 503"),
+ }
+ w := &CustomerBackupRunnerWorker{
+ db: db, store: store,
+ pgDump: &fakePgDump{err: errors.New("pg_dump down")},
+ bucket: "b", prefix: "p", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: backupBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRunnerJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+// ────────────────────────────────────────────────────────────────────
+// customer_restore_runner.go — remaining branches
+// ────────────────────────────────────────────────────────────────────
+
+// TestRestoreRunner_ClaimDBError_LogsContinues — non-ErrNoRows error on
+// the claim UPDATE is logged + skipped.
+func TestRestoreRunner_ClaimDBError_LogsContinues(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+
+ restoreID := "rrrrrrr0-1111-2222-3333-444444444444"
+ resID := "22222222-2222-2222-2222-222222222222"
+ teamID := uuid.New()
+ enc := encryptForTest(t, "postgres://u:p@host/db")
+
+ mock.ExpectExec(`UPDATE resource_restores\s+SET status\s+= 'failed'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT rr\.id::text`).
+ WithArgs(restoreBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "backup_id", "s3_key", "sha256",
+ "connection_url", "resource_type", "token", "team_id",
+ }).AddRow(restoreID, resID, "bk", "backups/tk/abc.dump.gz", nil, enc, "postgres", "tk", teamID))
+ mock.ExpectQuery(`UPDATE resource_restores\s+SET status = 'running'`).
+ WithArgs(restoreID).
+ WillReturnError(errors.New("claim down"))
+
+ w := &CustomerRestoreRunnerWorker{
+ db: db, store: newFakeBackupStore(), pgRestore: &fakePgRestore{},
+ bucket: "instant-shared", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: restoreBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRestoreJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+// TestRestoreRunner_FinalizeUpdateError_LogsAndReturns — pg_restore +
+// gzip succeed but the finalize UPDATE errors.
+func TestRestoreRunner_FinalizeUpdateError_LogsAndReturns(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+
+ restoreID := "rrrrrrr0-1111-2222-3333-444444444444"
+ resID := "22222222-2222-2222-2222-222222222222"
+ teamID := uuid.New()
+ enc := encryptForTest(t, "postgres://u:p@host/db")
+ s3Key := "backups/tk/ok.dump.gz"
+ store := newFakeBackupStore()
+ store.objects["instant-shared/"+s3Key] = gzipFor(t, []byte("p"))
+
+ mock.ExpectExec(`UPDATE resource_restores\s+SET status\s+= 'failed'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT rr\.id::text`).
+ WithArgs(restoreBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "backup_id", "s3_key", "sha256",
+ "connection_url", "resource_type", "token", "team_id",
+ }).AddRow(restoreID, resID, "bk", s3Key, nil, enc, "postgres", "tk", teamID))
+ mock.ExpectQuery(`UPDATE resource_restores\s+SET status = 'running'`).
+ WithArgs(restoreID).
+ WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(restoreID))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`UPDATE resource_restores\s+SET status = 'ok'`).
+ WithArgs(restoreID).
+ WillReturnError(errors.New("finalize down"))
+
+ w := &CustomerRestoreRunnerWorker{
+ db: db, store: store, pgRestore: &fakePgRestore{},
+ bucket: "instant-shared", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: restoreBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRestoreJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+// TestRestoreRunner_WriteAudit_DBError_LogsContinues — restore.started
+// audit INSERT errors; the rest of the row continues.
+func TestRestoreRunner_WriteAudit_DBError_LogsContinues(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+
+ restoreID := "rrrrrrr0-1111-2222-3333-444444444444"
+ resID := "22222222-2222-2222-2222-222222222222"
+ teamID := uuid.New()
+ enc := encryptForTest(t, "postgres://u:p@host/db")
+ s3Key := "backups/tk/ok.dump.gz"
+ store := newFakeBackupStore()
+ store.objects["instant-shared/"+s3Key] = gzipFor(t, []byte("p"))
+
+ mock.ExpectExec(`UPDATE resource_restores\s+SET status\s+= 'failed'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT rr\.id::text`).
+ WithArgs(restoreBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "backup_id", "s3_key", "sha256",
+ "connection_url", "resource_type", "token", "team_id",
+ }).AddRow(restoreID, resID, "bk", s3Key, nil, enc, "postgres", "tk", teamID))
+ mock.ExpectQuery(`UPDATE resource_restores\s+SET status = 'running'`).
+ WithArgs(restoreID).
+ WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(restoreID))
+ // started audit INSERT errors.
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnError(errors.New("audit down"))
+ mock.ExpectExec(`UPDATE resource_restores\s+SET status = 'ok'`).
+ WithArgs(restoreID).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+
+ w := &CustomerRestoreRunnerWorker{
+ db: db, store: store, pgRestore: &fakePgRestore{},
+ bucket: "instant-shared", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: restoreBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRestoreJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+// TestRestoreRunner_MarkRestoreFailed_DBError_LogsContinues — markFailed
+// UPDATE errors; the audit row still fires.
+func TestRestoreRunner_MarkRestoreFailed_DBError_LogsContinues(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+
+ restoreID := "rrrrrrr0-1111-2222-3333-444444444444"
+ resID := "22222222-2222-2222-2222-222222222222"
+ teamID := uuid.New()
+
+ mock.ExpectExec(`UPDATE resource_restores\s+SET status\s+= 'failed'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT rr\.id::text`).
+ WithArgs(restoreBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "backup_id", "s3_key", "sha256",
+ "connection_url", "resource_type", "token", "team_id",
+ }).AddRow(restoreID, resID, "bk", nil, nil, nil, "postgres", "tk", teamID))
+ mock.ExpectQuery(`UPDATE resource_restores\s+SET status = 'running'`).
+ WithArgs(restoreID).
+ WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(restoreID))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ // markRestoreFailed UPDATE errors.
+ mock.ExpectExec(`UPDATE resource_restores\s+SET status = 'failed'`).
+ WithArgs(restoreID, sqlmock.AnyArg()).
+ WillReturnError(errors.New("mark failed down"))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+
+ w := &CustomerRestoreRunnerWorker{
+ db: db, store: newFakeBackupStore(), pgRestore: &fakePgRestore{},
+ bucket: "instant-shared", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: restoreBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRestoreJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+// ────────────────────────────────────────────────────────────────────
+// platform_db_backup.go — remaining branches
+// ────────────────────────────────────────────────────────────────────
+
+// TestPlatformDBBackup_WriteAudit_InsertError_LogsContinues — the
+// started-audit INSERT errors; the rest of the pipeline runs unaffected.
+func TestPlatformDBBackup_WriteAudit_InsertError_LogsContinues(t *testing.T) {
+ db, mock, err := sqlmock.New(
+ sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp),
+ sqlmock.MonitorPingsOption(false),
+ )
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+
+ expectAdvisoryLockAcquired(mock)
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs("system", "platform_backup.started", sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnError(errors.New("audit down"))
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs("system", "platform_backup.succeeded", sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ expectAdvisoryUnlock(mock)
+
+ dumper := &fakePgDumper{payload: []byte("ok")}
+ s3 := newFakeS3()
+ now := time.Date(2026, 5, 13, 2, 0, 0, 0, time.UTC)
+ w := newTestWorker(t, mock, db, dumper, s3, now)
+ if err := w.Work(context.Background(), fakePlatformBackupJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+// TestPlatformDBBackup_SweepDeleteError_NonFatal — Delete on a retention
+// victim fails; the sweep continues and Work still returns nil.
+func TestPlatformDBBackup_SweepDeleteError_NonFatal(t *testing.T) {
+ db, mock, err := sqlmock.New(
+ sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp),
+ sqlmock.MonitorPingsOption(false),
+ )
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+
+ expectAdvisoryLockAcquired(mock)
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs("system", "platform_backup.started", sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs("system", "platform_backup.succeeded", sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ expectAdvisoryUnlock(mock)
+
+ dumper := &fakePgDumper{payload: []byte("ok")}
+ s3 := newFakeS3()
+ // List returns a victim; Delete on it errors.
+ s3.listResult = []string{"platform-backups/2024-01-01/platform.dump.gz"}
+ s3.deleteErr = errors.New("delete 500")
+ now := time.Date(2026, 5, 13, 2, 0, 0, 0, time.UTC)
+ w := newTestWorker(t, mock, db, dumper, s3, now)
+ if err := w.Work(context.Background(), fakePlatformBackupJob()); err != nil {
+ t.Errorf("Work: sweep delete failure must be non-fatal: %v", err)
+ }
+}
+
+// TestPlatformDBBackup_DumpError_DeletePartialObject — the dumper errors;
+// the worker tries to delete the partial object (best-effort).
+func TestPlatformDBBackup_DumpError_DeletePartialObject(t *testing.T) {
+ db, mock, err := sqlmock.New(
+ sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp),
+ sqlmock.MonitorPingsOption(false),
+ )
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+
+ expectAdvisoryLockAcquired(mock)
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs("system", "platform_backup.started", sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs("system", "platform_backup.failed", sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ expectAdvisoryUnlock(mock)
+
+ // When the dumper errors, Work calls pw.CloseWithError(dumpErr), so the
+ // error propagates through the io.Pipe to the uploader's Read. The
+ // uploader therefore also fails (uploadErr != nil), and Work's
+ // partial-object delete branch — guarded by (uploadErr == nil &&
+ // dumpErr != nil) — is NOT taken. The observable contract on a dump
+ // error is: Work returns a non-nil error wrapping the dump failure,
+ // writes the `platform_backup.failed` audit row, and uploads no usable
+ // object. The cleanup-delete branch is only reachable if a future
+ // rewiring lets an upload succeed while the dump fails; this test pins
+ // today's behavior so that change is a conscious one.
+ dumper := &fakePgDumper{err: errors.New("pg_dump down")}
+ s3 := newFakeS3()
+ now := time.Date(2026, 5, 13, 2, 0, 0, 0, time.UTC)
+ w := newTestWorker(t, mock, db, dumper, s3, now)
+ err = w.Work(context.Background(), fakePlatformBackupJob())
+ if err == nil {
+ t.Fatal("expected error")
+ }
+ if !strings.Contains(err.Error(), "pg_dump") {
+ t.Errorf("error should wrap the dump failure; got %v", err)
+ }
+ // No usable object was uploaded.
+ if _, ok := s3.uploaded[fmt.Sprintf("platform-backups/2026-05-13/%s", platformBackupObjectName)]; ok {
+ t.Errorf("no object should be uploaded on dump failure")
+ }
+}
+
+// TestPlatformDBBackup_AcquireConnError — db.Conn returns an error before
+// the advisory lock. Work surfaces a non-nil error.
+func TestPlatformDBBackup_AcquireConnError(t *testing.T) {
+ db, _, err := sqlmock.New(sqlmock.MonitorPingsOption(false))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ // Closing the DB before Work runs forces db.Conn(...) to error.
+ db.Close()
+
+ w := NewPlatformDBBackupWorker(PlatformDBBackupConfig{
+ DB: db,
+ DatabaseURL: "postgres://x",
+ S3: newFakeS3(),
+ Bucket: "b",
+ InnerPrefix: "platform-backups/",
+ Now: fixedClock(time.Date(2026, 5, 13, 2, 0, 0, 0, time.UTC)),
+ })
+ if err := w.Work(context.Background(), fakePlatformBackupJob()); err == nil {
+ t.Fatal("expected error from closed DB")
+ }
+}
+
+// TestComputeKeepSet_UnparseableDateInPath — defensive: a key with a
+// "date-shaped" path segment whose Parse fails (e.g. 2026-99-99) is
+// conservatively kept.
+func TestComputeKeepSet_UnparseableDateInPath(t *testing.T) {
+ now := time.Date(2026, 5, 13, 0, 0, 0, 0, time.UTC)
+ keys := []string{"platform-backups/2026-99-99/platform.dump.gz"}
+ keep := computeKeepSet(keys, now, 30, 12)
+ if !keep[keys[0]] {
+ t.Error("unparseable-date key should be kept (defensive)")
+ }
+}
+
+// ────────────────────────────────────────────────────────────────────
+// finalizeDigest — null hash path
+// ────────────────────────────────────────────────────────────────────
+
+// TestFinalizeDigest_NilOnError — both error paths return empty so the
+// finalize UPDATE writes NULL.
+func TestFinalizeDigest_NilOnError(t *testing.T) {
+ h := makeSHA256ForTest()
+ _, _ = h.Write([]byte("abc"))
+ if got := finalizeDigest(h, errors.New("dump"), nil); got != "" {
+ t.Errorf("dumpErr should return empty, got %q", got)
+ }
+ if got := finalizeDigest(h, nil, errors.New("up")); got != "" {
+ t.Errorf("upErr should return empty, got %q", got)
+ }
+ got := finalizeDigest(h, nil, nil)
+ if got == "" {
+ t.Errorf("happy path should return non-empty hex")
+ }
+ if _, err := hex.DecodeString(got); err != nil {
+ t.Errorf("hash output is not valid hex: %v", err)
+ }
+}
+
+// makeSHA256ForTest is a tiny helper so this test file owns its sha256
+// usage (avoiding "imported and not used" diagnostics under future edits).
+func makeSHA256ForTest() hash.Hash { return sha256.New() }
diff --git a/internal/jobs/coverage_gaps_test.go b/internal/jobs/coverage_gaps_test.go
new file mode 100644
index 0000000..78a974f
--- /dev/null
+++ b/internal/jobs/coverage_gaps_test.go
@@ -0,0 +1,1552 @@
+package jobs
+
+// coverage_gaps_test.go — fills the remaining sub-95% branches on the
+// backup + prober job files that the broader backup_extra_test.go /
+// coverage_misc_test.go suites left uncovered:
+//
+// - real_prober.go probe{Postgres,Redis,Mongo,Queue} SUCCESS returns,
+// driven against the live docker backends (skipped if unreachable).
+// - platform_db_backup_s3.go minioS3.{Upload,Delete} SUCCESS returns,
+// driven against an httptest server that mimics an S3 200/204.
+// - geodb.go Work happy path (download → extract → rename), driven
+// against an httptest server serving a real gzipped tarball.
+
+import (
+ "archive/tar"
+ "bytes"
+ "compress/gzip"
+ "context"
+ "database/sql"
+ "errors"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ sqlmock "github.com/DATA-DOG/go-sqlmock"
+ "github.com/google/uuid"
+ minio "github.com/minio/minio-go/v7"
+ "github.com/minio/minio-go/v7/pkg/credentials"
+ "github.com/riverqueue/river"
+ "github.com/riverqueue/river/rivertype"
+
+ "instant.dev/worker/internal/provisioner"
+)
+
+// dialable reports whether host:port accepts a TCP connection within 1s.
+func dialable(t *testing.T, hostport string) bool {
+ t.Helper()
+ c, err := net.DialTimeout("tcp", hostport, time.Second)
+ if err != nil {
+ return false
+ }
+ _ = c.Close()
+ return true
+}
+
+// ────────────────────────────────────────────────────────────────────
+// real_prober.go — probe* SUCCESS returns against live docker backends
+// ────────────────────────────────────────────────────────────────────
+
+func TestProber_ProbePostgres_LiveReachable(t *testing.T) {
+ if !dialable(t, "127.0.0.1:5432") {
+ t.Skip("postgres not reachable on 127.0.0.1:5432")
+ }
+ p := &realProber{httpClient: &http.Client{Timeout: 5 * time.Second}}
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ out, err := p.probePostgres(ctx, "postgres://postgres:postgres@127.0.0.1:5432/postgres?sslmode=disable")
+ if out != ProbeReachable || err != nil {
+ t.Fatalf("expected ProbeReachable, got %v err=%v", out, err)
+ }
+}
+
+func TestProber_ProbeRedis_LiveReachable(t *testing.T) {
+ if !dialable(t, "127.0.0.1:6379") {
+ t.Skip("redis not reachable on 127.0.0.1:6379")
+ }
+ p := &realProber{httpClient: &http.Client{Timeout: 5 * time.Second}}
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ out, err := p.probeRedis(ctx, "redis://127.0.0.1:6379")
+ if out != ProbeReachable || err != nil {
+ t.Fatalf("expected ProbeReachable, got %v err=%v", out, err)
+ }
+}
+
+func TestProber_ProbeMongo_LiveReachable(t *testing.T) {
+ if !dialable(t, "127.0.0.1:27017") {
+ t.Skip("mongodb not reachable on 127.0.0.1:27017")
+ }
+ p := &realProber{httpClient: &http.Client{Timeout: 5 * time.Second}}
+ ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second)
+ defer cancel()
+ out, err := p.probeMongo(ctx, "mongodb://127.0.0.1:27017")
+ if out != ProbeReachable || err != nil {
+ t.Fatalf("expected ProbeReachable, got %v err=%v", out, err)
+ }
+}
+
+func TestProber_ProbeQueue_LiveReachable(t *testing.T) {
+ // probeQueue builds http://:8222/healthz from the nats:// URL.
+ // The test-nats container exposes its monitoring endpoint on 8222.
+ if !dialable(t, "127.0.0.1:8222") {
+ t.Skip("nats monitoring not reachable on 127.0.0.1:8222")
+ }
+ p := &realProber{httpClient: &http.Client{Timeout: 5 * time.Second}}
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ out, err := p.probeQueue(ctx, "nats://127.0.0.1:4222")
+ if out != ProbeReachable || err != nil {
+ t.Fatalf("expected ProbeReachable, got %v err=%v", out, err)
+ }
+}
+
+func TestProber_ProbeQueue_NATSUnhealthyStatus(t *testing.T) {
+ // A monitoring endpoint that answers but with a non-200 status must
+ // surface ProbeUnreachable (the "NATS unhealthy (HTTP %d)" branch).
+ // We can't redirect the fixed :8222 port, so drive the http client
+ // directly through a stub by pointing probeQueue at a host whose
+ // :8222 returns 503. httptest binds a random port, so instead assert
+ // the status-code branch via probeStorage's sibling logic is covered
+ // elsewhere; here we cover the !=200 path using a server forced onto
+ // the loopback monitoring port is infeasible — fall back to verifying
+ // the parse+dispatch on a host that refuses :8222.
+ p := &realProber{httpClient: &http.Client{Timeout: 1 * time.Second}}
+ ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+ defer cancel()
+ // 127.0.0.1:8222 path is covered by the healthy test; use a
+ // guaranteed-unroutable host to exercise the GET-error branch.
+ out, _ := p.probeQueue(ctx, "nats://192.0.2.1:4222")
+ if out != ProbeUnreachable {
+ t.Fatalf("expected ProbeUnreachable for unroutable nats host, got %v", out)
+ }
+}
+
+func TestProber_ProbePostgres_OpenError(t *testing.T) {
+ // An unparseable Postgres DSN makes sql.Open (lib/pq) fail at open
+ // time → the sql.Open error branch.
+ p := &realProber{httpClient: &http.Client{Timeout: time.Second}}
+ out, err := p.probePostgres(context.Background(), "postgres://%zz")
+ if out != ProbeUnreachable || err == nil {
+ t.Fatalf("expected ProbeUnreachable+err, got %v / %v", out, err)
+ }
+}
+
+func TestProber_ProbePostgres_PingError(t *testing.T) {
+ // A well-formed DSN pointing at an unroutable host fails the SELECT 1
+ // ping → the "SELECT 1" error branch.
+ p := &realProber{httpClient: &http.Client{Timeout: time.Second}}
+ ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+ defer cancel()
+ out, err := p.probePostgres(ctx, "postgres://u:p@192.0.2.1:5432/db?sslmode=disable&connect_timeout=1")
+ if out != ProbeUnreachable || err == nil {
+ t.Fatalf("expected ProbeUnreachable+err, got %v / %v", out, err)
+ }
+}
+
+func TestProber_ProbeRedis_ParseURLError(t *testing.T) {
+ p := &realProber{httpClient: &http.Client{Timeout: time.Second}}
+ out, err := p.probeRedis(context.Background(), "not-a-redis-url")
+ if out != ProbeUnreachable || err == nil {
+ t.Fatalf("expected ProbeUnreachable+err, got %v / %v", out, err)
+ }
+}
+
+func TestProber_ProbeMongo_ConnectError(t *testing.T) {
+ // An invalid mongo URI fails ApplyURI/Connect.
+ p := &realProber{httpClient: &http.Client{Timeout: time.Second}}
+ out, err := p.probeMongo(context.Background(), "mongodb://[::bad")
+ if out != ProbeUnreachable || err == nil {
+ t.Fatalf("expected ProbeUnreachable+err, got %v / %v", out, err)
+ }
+}
+
+func TestProber_ProbeStorage_BuildRequestError(t *testing.T) {
+ // A normalized URL containing a control character makes
+ // http.NewRequestWithContext fail.
+ p := &realProber{httpClient: &http.Client{Timeout: time.Second}}
+ out, err := p.probeStorage(context.Background(), "http://exa\x7fmple.com/b")
+ if out != ProbeUnreachable || err == nil {
+ t.Fatalf("expected ProbeUnreachable+err, got %v / %v", out, err)
+ }
+}
+
+func TestProber_ProbeQueue_Non200_Unreachable(t *testing.T) {
+ // Drive probeQueue at a host whose :8222 monitoring endpoint answers
+ // with a non-200 status. We host an httptest server, then point the
+ // nats URL at its host and override the prober's httpClient transport
+ // so the fixed :8222 target is rerouted to the test server.
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusServiceUnavailable)
+ }))
+ defer srv.Close()
+ rt := &rerouteTransport{target: srv.Listener.Addr().String()}
+ p := &realProber{httpClient: &http.Client{Timeout: 2 * time.Second, Transport: rt}}
+ out, err := p.probeQueue(context.Background(), "nats://127.0.0.1:4222")
+ if out != ProbeUnreachable || err == nil || !strings.Contains(err.Error(), "unhealthy") {
+ t.Fatalf("expected unhealthy ProbeUnreachable, got %v / %v", out, err)
+ }
+}
+
+// rerouteTransport rewrites every request's host:port to target so a probe
+// that builds a fixed-port URL can be pointed at an httptest server.
+type rerouteTransport struct{ target string }
+
+func (rt *rerouteTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+ req.URL.Host = rt.target
+ return http.DefaultTransport.RoundTrip(req)
+}
+
+func TestProber_Probe_UnknownType_Skip(t *testing.T) {
+ // An unknown resource_type routes to the switch default → ProbeSkip.
+ p := &realProber{httpClient: &http.Client{Timeout: time.Second}}
+ out, err := p.Probe(context.Background(), "quantum-db", "plaintext://x")
+ if out != ProbeSkip || err != nil {
+ t.Fatalf("expected ProbeSkip, got %v / %v", out, err)
+ }
+}
+
+func TestProber_ProbeStorage_LiveHEAD(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ }))
+ defer srv.Close()
+ p := &realProber{httpClient: &http.Client{Timeout: 5 * time.Second}}
+ out, err := p.probeStorage(context.Background(), srv.URL+"/bucket")
+ if out != ProbeReachable || err != nil {
+ t.Fatalf("expected ProbeReachable, got %v err=%v", out, err)
+ }
+}
+
+// ────────────────────────────────────────────────────────────────────
+// platform_db_backup_s3.go — minioS3 Upload / Delete SUCCESS returns
+// ────────────────────────────────────────────────────────────────────
+
+// newLiveMinioS3 dials the local test-minio container (minioadmin) and
+// returns the production wrapper plus a freshly-created bucket name. The
+// minio-go client does a bucket-location handshake on the first call that
+// a 4-line httptest stub cannot satisfy, so the SUCCESS returns of
+// Upload/List/Delete are exercised against the real backend (skipped when
+// the container isn't running).
+func newLiveMinioS3(t *testing.T) (*minioS3, string) {
+ t.Helper()
+ if !dialable(t, "127.0.0.1:9100") {
+ t.Skip("test-minio not reachable on 127.0.0.1:9100")
+ }
+ cli, err := minio.New("127.0.0.1:9100", &minio.Options{
+ Creds: credentials.NewStaticV4("minioadmin", "minioadmin", ""),
+ Secure: false,
+ })
+ if err != nil {
+ t.Fatalf("minio.New: %v", err)
+ }
+ bucket := fmt.Sprintf("cov-%d", time.Now().UnixNano())
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ if err := cli.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil {
+ t.Fatalf("MakeBucket: %v", err)
+ }
+ t.Cleanup(func() {
+ _ = cli.RemoveBucket(context.Background(), bucket)
+ })
+ return &minioS3{client: cli}, bucket
+}
+
+func TestMinioS3_UploadListDelete_Success(t *testing.T) {
+ s3, bucket := newLiveMinioS3(t)
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+
+ key := "platform-backups/2026-05-13/platform.dump.gz"
+ payload := []byte("payload-bytes")
+ // Upload SUCCESS return.
+ if err := s3.Upload(ctx, bucket, key, bytes.NewReader(payload), int64(len(payload))); err != nil {
+ t.Fatalf("Upload success path: %v", err)
+ }
+ // List SUCCESS return — the uploaded key must be enumerated.
+ keys, err := s3.List(ctx, bucket, "platform-backups/")
+ if err != nil {
+ t.Fatalf("List success path: %v", err)
+ }
+ found := false
+ for _, k := range keys {
+ if k == key {
+ found = true
+ }
+ }
+ if !found {
+ t.Fatalf("uploaded key not listed; got %v", keys)
+ }
+ // Delete SUCCESS return.
+ if err := s3.Delete(ctx, bucket, key); err != nil {
+ t.Fatalf("Delete success path: %v", err)
+ }
+ // And a streaming (size=-1, multipart) upload also returns nil.
+ if err := s3.Upload(ctx, bucket, "stream/obj", bytes.NewReader(payload), -1); err != nil {
+ t.Fatalf("streaming Upload success path: %v", err)
+ }
+ _ = s3.Delete(ctx, bucket, "stream/obj")
+}
+
+// ────────────────────────────────────────────────────────────────────
+// customer_restore_runner.go — Work scan/rows-error + ctx-cancel branches
+// ────────────────────────────────────────────────────────────────────
+
+func TestRestoreRunner_Work_RowsError_ReturnsError(t *testing.T) {
+ // A RowError attached to the result set surfaces as rows.Err() after
+ // the scan loop, hitting the `rows error` return branch.
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ mock.ExpectExec(`UPDATE resource_restores\s+SET status\s+= 'failed'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ rows := sqlmock.NewRows([]string{
+ "id", "resource_id", "backup_id", "s3_key", "sha256",
+ "connection_url", "resource_type", "token", "team_id",
+ }).AddRow("rid", "resid", "bk", "k", nil, "url", "postgres", "tok", uuid.New()).
+ RowError(0, errors.New("rows boom"))
+ mock.ExpectQuery(`SELECT rr\.id::text`).WithArgs(restoreBatchSize).WillReturnRows(rows)
+
+ w := &CustomerRestoreRunnerWorker{
+ db: db, store: newFakeBackupStore(), pgRestore: &fakePgRestore{},
+ bucket: "instant-shared", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: restoreBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRestoreJob()); err == nil ||
+ !strings.Contains(err.Error(), "rows error") {
+ t.Fatalf("expected rows-error, got %v", err)
+ }
+}
+
+func TestRestoreRunner_Work_ScanError_SkipsRow(t *testing.T) {
+ // A row whose team_id column holds a value that cannot scan into
+ // uuid.NullUUID triggers a per-row Scan error → the `scan_failed`
+ // continue branch. With that row skipped and no others, Work returns
+ // nil cleanly.
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ mock.ExpectExec(`UPDATE resource_restores\s+SET status\s+= 'failed'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT rr\.id::text`).WithArgs(restoreBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "backup_id", "s3_key", "sha256",
+ "connection_url", "resource_type", "token", "team_id",
+ }).AddRow("rid", "resid", "bk", "k", nil, "url", "postgres", "tok", "not-a-uuid"))
+
+ w := &CustomerRestoreRunnerWorker{
+ db: db, store: newFakeBackupStore(), pgRestore: &fakePgRestore{},
+ bucket: "instant-shared", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: restoreBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRestoreJob()); err != nil {
+ t.Fatalf("Work should skip the unscannable row and return nil, got %v", err)
+ }
+}
+
+// errReadStore is a BackupObjectStore whose Download returns a reader that
+// errors after the first Read — exercising the restore runner's "S3 read
+// failed" branch (the io.Copy from the downloaded stream).
+type errReadStore struct{}
+
+func (errReadStore) Upload(context.Context, string, string, io.Reader) (int64, error) {
+ return 0, nil
+}
+func (errReadStore) Download(context.Context, string, string) (io.ReadCloser, error) {
+ return io.NopCloser(errReader{}), nil
+}
+func (errReadStore) DeleteObject(context.Context, string, string) error { return nil }
+
+type errReader struct{}
+
+func (errReader) Read([]byte) (int, error) { return 0, errors.New("mid-stream S3 read boom") }
+
+func TestRestoreRunner_ProcessRestore_S3ReadError_MarksFailed(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ restoreID := "rrrrrrr0-1111-2222-3333-444444444444"
+ resID := "22222222-2222-2222-2222-222222222222"
+ teamID := uuid.New()
+ encConn := encryptForTest(t, "postgres://u:p@host/db")
+ mock.ExpectExec(`UPDATE resource_restores\s+SET status\s+= 'failed'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT rr\.id::text`).WithArgs(restoreBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "backup_id", "s3_key", "sha256",
+ "connection_url", "resource_type", "token", "team_id",
+ }).AddRow(restoreID, resID, "bk", "k", nil, encConn, "postgres", "tk", teamID))
+ mock.ExpectQuery(`UPDATE resource_restores\s+SET status = 'running'`).
+ WithArgs(restoreID).WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(restoreID))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`UPDATE resource_restores\s+SET status = 'failed'`).
+ WithArgs(restoreID, sqlmock.AnyArg()).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+
+ w := &CustomerRestoreRunnerWorker{
+ db: db, store: errReadStore{}, pgRestore: &fakePgRestore{},
+ bucket: "instant-shared", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: restoreBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRestoreJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+// NOTE: the customer_restore_runner Work ctx-cancelled-mid-batch branch
+// (the per-row `case <-ctx.Done()`) is not hermetically reachable: under
+// sqlmock a pre-cancelled context fails the pending-row SELECT (returns
+// `context canceled`) before the per-row loop is entered, and a real DB
+// would do the same. This mirrors the documented limitation in
+// customer_backup_runner_test.go ("a pre-cancelled ctx fails the SELECT
+// before the per-row ctx.Done() check is reachable"). The branch is left
+// uncovered by design rather than via a flaky time-raced cancellation.
+
+func TestRestoreRunner_ProcessRestore_TempCreateError_MarksFailed(t *testing.T) {
+ // Point TMPDIR at a non-existent directory so os.CreateTemp fails after
+ // a successful S3 download, hitting the "create temp file" branch.
+ t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "does-not-exist"))
+
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ restoreID := "rrrrrrr0-1111-2222-3333-444444444444"
+ resID := "22222222-2222-2222-2222-222222222222"
+ teamID := uuid.New()
+ encConn := encryptForTest(t, "postgres://u:p@host/db")
+ store := newFakeBackupStore()
+ store.objects["instant-shared/k"] = gzipFor(t, []byte("payload"))
+
+ mock.ExpectExec(`UPDATE resource_restores\s+SET status\s+= 'failed'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT rr\.id::text`).WithArgs(restoreBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "backup_id", "s3_key", "sha256",
+ "connection_url", "resource_type", "token", "team_id",
+ }).AddRow(restoreID, resID, "bk", "k", nil, encConn, "postgres", "tk", teamID))
+ mock.ExpectQuery(`UPDATE resource_restores\s+SET status = 'running'`).
+ WithArgs(restoreID).WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(restoreID))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`UPDATE resource_restores\s+SET status = 'failed'`).
+ WithArgs(restoreID, sqlmock.AnyArg()).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+
+ w := &CustomerRestoreRunnerWorker{
+ db: db, store: store, pgRestore: &fakePgRestore{},
+ bucket: "instant-shared", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: restoreBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRestoreJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+// ────────────────────────────────────────────────────────────────────
+// backup_s3.go — minioBackupStore Upload / Download / DeleteObject success
+// ────────────────────────────────────────────────────────────────────
+
+// newLiveMinioBackupStore mirrors newLiveMinioS3 but returns the
+// minioBackupStore wrapper used by the customer backup runner.
+func newLiveMinioBackupStore(t *testing.T) (*minioBackupStore, string) {
+ t.Helper()
+ if !dialable(t, "127.0.0.1:9100") {
+ t.Skip("test-minio not reachable on 127.0.0.1:9100")
+ }
+ store, err := NewMinIOBackupStore("http://127.0.0.1:9100", "minioadmin", "minioadmin")
+ if err != nil {
+ t.Fatalf("NewMinIOBackupStore: %v", err)
+ }
+ bucket := fmt.Sprintf("covbk-%d", time.Now().UnixNano())
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ if err := store.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil {
+ t.Fatalf("MakeBucket: %v", err)
+ }
+ t.Cleanup(func() { _ = store.client.RemoveBucket(context.Background(), bucket) })
+ return store, bucket
+}
+
+func TestMinioBackupStore_UploadDownloadDelete_Success(t *testing.T) {
+ store, bucket := newLiveMinioBackupStore(t)
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+
+ key := backupObjectKey("backups/", "tok-cov", "bk-123")
+ payload := []byte("gzipped-dump-bytes")
+ // Upload SUCCESS — returns the persisted size.
+ n, err := store.Upload(ctx, bucket, key, bytes.NewReader(payload))
+ if err != nil {
+ t.Fatalf("Upload success path: %v", err)
+ }
+ if n != int64(len(payload)) {
+ t.Errorf("Upload size = %d, want %d", n, len(payload))
+ }
+ // Download SUCCESS — the returned ReadCloser yields the bytes back.
+ rc, err := store.Download(ctx, bucket, key)
+ if err != nil {
+ t.Fatalf("Download success path: %v", err)
+ }
+ got, _ := io.ReadAll(rc)
+ _ = rc.Close()
+ if !bytes.Equal(got, payload) {
+ t.Errorf("Download bytes mismatch: %q vs %q", got, payload)
+ }
+ // DeleteObject SUCCESS.
+ if err := store.DeleteObject(ctx, bucket, key); err != nil {
+ t.Fatalf("DeleteObject success path: %v", err)
+ }
+}
+
+// ────────────────────────────────────────────────────────────────────
+// customer_backup_runner.go — pg_dump panic + refund-failed WARN branch
+// ────────────────────────────────────────────────────────────────────
+
+// panicPgDump panics inside Run, exercising the pg_dump goroutine's
+// recover() boundary in processBackup.
+type panicPgDump struct{}
+
+func (panicPgDump) Run(context.Context, string, io.Writer) error { panic("pg_dump exploded") }
+
+func TestRunner_ProcessBackup_PgDumpPanic_Recovered(t *testing.T) {
+ db, mock, err := sqlmock.New()
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ backupID := "11111111-1111-1111-1111-111111111111"
+ resID := "22222222-2222-2222-2222-222222222222"
+ enc := encryptForTest(t, "postgres://u:p@host/db")
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'pending'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT b.id::text`).WithArgs(backupBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "tier_at_backup", "backup_kind",
+ "token", "connection_url", "resource_type", "team_id",
+ }).AddRow(backupID, resID, "pro", "scheduled", "tk", enc, "postgres", uuid.New()))
+ mock.ExpectQuery(`UPDATE resource_backups\s+SET status = 'running'`).
+ WithArgs(backupID).WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(backupID))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'failed'`).
+ WithArgs(backupID, sqlmock.AnyArg()).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ for i := 0; i < 5; i++ {
+ mock.ExpectQuery(`SELECT id::text, s3_key`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "s3_key"}))
+ }
+
+ w := &CustomerBackupRunnerWorker{
+ db: db, store: newFakeBackupStore(), pgDump: panicPgDump{},
+ bucket: "b", prefix: "p", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: backupBatchSize,
+ }
+ // Work must not crash — the panic is recovered and the row marked failed.
+ if err := w.Work(context.Background(), fakeRunnerJob()); err != nil {
+ t.Fatalf("Work should recover the panic and return nil, got %v", err)
+ }
+}
+
+func TestRunner_ProcessBackup_ManualKind_RefundError_LoggedWarn(t *testing.T) {
+ // A manual-kind backup that fails AND whose refund call errors hits the
+ // refund_failed WARN branch (599).
+ db, mock, err := sqlmock.New()
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ backupID := "11111111-1111-1111-1111-111111111111"
+ resID := "22222222-2222-2222-2222-222222222222"
+ enc := encryptForTest(t, "postgres://u:p@host/db")
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'pending'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT b.id::text`).WithArgs(backupBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "tier_at_backup", "backup_kind",
+ "token", "connection_url", "resource_type", "team_id",
+ }).AddRow(backupID, resID, "pro", "manual", "tk", enc, "postgres", uuid.New()))
+ mock.ExpectQuery(`UPDATE resource_backups\s+SET status = 'running'`).
+ WithArgs(backupID).WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(backupID))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'failed'`).
+ WithArgs(backupID, sqlmock.AnyArg()).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ for i := 0; i < 5; i++ {
+ mock.ExpectQuery(`SELECT id::text, s3_key`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "s3_key"}))
+ }
+
+ // Refund endpoint returns 500 → refundManualBackupQuota returns an error.
+ apiSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusInternalServerError)
+ }))
+ defer apiSrv.Close()
+
+ w := (&CustomerBackupRunnerWorker{
+ db: db, store: newFakeBackupStore(),
+ pgDump: &fakePgDump{err: errors.New("pg_dump down")},
+ bucket: "b", prefix: "p", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: backupBatchSize,
+ }).WithRefundClient(apiSrv.URL, "refund-secret", &http.Client{Timeout: 5 * time.Second})
+ if err := w.Work(context.Background(), fakeRunnerJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+// ────────────────────────────────────────────────────────────────────
+// customer_backup_runner.go — refundManualBackupQuota HTTP paths
+// ────────────────────────────────────────────────────────────────────
+
+func TestRunner_RefundManualBackupQuota_Success(t *testing.T) {
+ // A wired apiBase/jwtSecret/apiCli + a 200 response exercises the
+ // build-request → sign-jwt → Do → 2xx success path.
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Header.Get("Authorization") == "" {
+ t.Error("missing bearer token")
+ }
+ w.WriteHeader(http.StatusOK)
+ _, _ = io.WriteString(w, `{"ok":true}`)
+ }))
+ defer srv.Close()
+
+ db, _, _ := sqlmock.New()
+ defer db.Close()
+ w := (&CustomerBackupRunnerWorker{db: db}).
+ WithRefundClient(srv.URL, "refund-secret", &http.Client{Timeout: 5 * time.Second})
+ if err := w.refundManualBackupQuota(uuid.New(), "bk-1"); err != nil {
+ t.Fatalf("refund success path: %v", err)
+ }
+}
+
+func TestRunner_RefundManualBackupQuota_Non2xx(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusForbidden)
+ _, _ = io.WriteString(w, "denied")
+ }))
+ defer srv.Close()
+
+ db, _, _ := sqlmock.New()
+ defer db.Close()
+ w := (&CustomerBackupRunnerWorker{db: db}).
+ WithRefundClient(srv.URL, "refund-secret", &http.Client{Timeout: 5 * time.Second})
+ if err := w.refundManualBackupQuota(uuid.New(), "bk-1"); err == nil ||
+ !strings.Contains(err.Error(), "api status 403") {
+ t.Fatalf("expected api-status error, got %v", err)
+ }
+}
+
+func TestRunner_RefundManualBackupQuota_RequestError(t *testing.T) {
+ // A server that is immediately closed → the apiCli.Do call errors
+ // (connection refused), exercising the api-request error branch.
+ srv := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
+ base := srv.URL
+ srv.Close()
+
+ db, _, _ := sqlmock.New()
+ defer db.Close()
+ w := (&CustomerBackupRunnerWorker{db: db}).
+ WithRefundClient(base, "refund-secret", &http.Client{Timeout: 2 * time.Second})
+ if err := w.refundManualBackupQuota(uuid.New(), "bk-1"); err == nil {
+ t.Fatal("expected request error against closed server")
+ }
+}
+
+// ────────────────────────────────────────────────────────────────────
+// resource_heartbeat.go — scan_failed + rows.Err branches
+// ────────────────────────────────────────────────────────────────────
+
+func TestHeartbeat_Work_ScanError_SkipsRow(t *testing.T) {
+ db, mock, err := sqlmock.New()
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ // id column holds a non-UUID → Scan into uuid.UUID errors → the
+ // scan_failed continue branch. With no usable candidates the sweep
+ // proceeds to the gauge query and returns nil.
+ mock.ExpectQuery(`FROM resources`).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "token", "resource_type", "connection_url",
+ "team_id_text", "degraded", "last_seen_at",
+ }).AddRow("not-a-uuid", uuid.New(), "postgres", "url", "", false, sql.NullTime{}))
+ mock.ExpectQuery(`SELECT resource_type, COUNT\(\*\)`).
+ WillReturnRows(sqlmock.NewRows([]string{"resource_type", "count"}))
+
+ w := NewResourceHeartbeatWorker(db, &localFakeProber{outcome: ProbeReachable})
+ if err := w.Work(context.Background(), localJob[ResourceHeartbeatArgs]()); err != nil {
+ t.Fatalf("Work should skip unscannable row, got %v", err)
+ }
+}
+
+func TestHeartbeat_Work_RowsError_ReturnsError(t *testing.T) {
+ db, mock, err := sqlmock.New()
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ rows := sqlmock.NewRows([]string{
+ "id", "token", "resource_type", "connection_url",
+ "team_id_text", "degraded", "last_seen_at",
+ }).AddRow(uuid.New(), uuid.New(), "postgres", "url", uuid.New().String(), false, sql.NullTime{}).
+ RowError(0, errors.New("rows boom"))
+ mock.ExpectQuery(`FROM resources`).WillReturnRows(rows)
+
+ w := NewResourceHeartbeatWorker(db, &localFakeProber{outcome: ProbeReachable})
+ if err := w.Work(context.Background(), localJob[ResourceHeartbeatArgs]()); err == nil ||
+ !strings.Contains(err.Error(), "rows error") {
+ t.Fatalf("expected rows-error, got %v", err)
+ }
+}
+
+// ────────────────────────────────────────────────────────────────────
+// customer_backup_scheduler.go — scan_failed + rows.Err branches
+// ────────────────────────────────────────────────────────────────────
+
+func TestScheduler_Work_ScanError_SkipsRow(t *testing.T) {
+ db, mock, err := sqlmock.New()
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ // team_id holds a non-UUID → the Scan into uuid type errors → the
+ // scan_failed continue branch fires; with no usable candidates the
+ // sweep completes cleanly.
+ mock.ExpectQuery(`SELECT r.id::text, r.tier, r.team_id`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "tier", "team_id"}).
+ AddRow("fffffff0-1111-2222-3333-444444444444", "pro", "not-a-uuid"))
+
+ w := NewCustomerBackupSchedulerWorker(db)
+ w.now = func() time.Time { return time.Date(2026, 5, 13, 14, 0, 0, 0, time.UTC) }
+ if err := w.Work(context.Background(), fakeSchedulerJob()); err != nil {
+ t.Fatalf("Work should skip unscannable row, got %v", err)
+ }
+}
+
+func TestScheduler_Work_RowsError_ReturnsError(t *testing.T) {
+ db, mock, err := sqlmock.New()
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ rows := sqlmock.NewRows([]string{"id", "tier", "team_id"}).
+ AddRow("fffffff0-1111-2222-3333-444444444444", "pro", uuid.New()).
+ RowError(0, errors.New("rows boom"))
+ mock.ExpectQuery(`SELECT r.id::text, r.tier, r.team_id`).WillReturnRows(rows)
+
+ w := NewCustomerBackupSchedulerWorker(db)
+ w.now = func() time.Time { return time.Date(2026, 5, 13, 14, 0, 0, 0, time.UTC) }
+ if err := w.Work(context.Background(), fakeSchedulerJob()); err == nil ||
+ !strings.Contains(err.Error(), "rows error") {
+ t.Fatalf("expected rows-error, got %v", err)
+ }
+}
+
+// ────────────────────────────────────────────────────────────────────
+// team_deletion_executor.go — processTeam error branches
+// ────────────────────────────────────────────────────────────────────
+
+// teamDelCandCols / teamDelResCols mirror the executor's scan projections.
+var teamDelCandCols = []string{"id", "deletion_requested_at"}
+var teamDelResCols = []string{"id", "token", "resource_type", "provider_resource_id"}
+
+func seedTeamDeletionCandidate(mock sqlmock.Sqlmock, teamID uuid.UUID) {
+ mock.ExpectQuery(`FROM teams\s+WHERE`).
+ WillReturnRows(sqlmock.NewRows(teamDelCandCols).
+ AddRow(teamID, time.Now().UTC().Add(-31*24*time.Hour)))
+ mock.ExpectExec(`UPDATE teams\s+SET status = 'deletion_pending'`).
+ WithArgs(teamID).WillReturnResult(sqlmock.NewResult(0, 1))
+}
+
+func TestTeamDeletion_MarkPending_Error(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ teamID := uuid.New()
+ mock.ExpectQuery(`FROM teams\s+WHERE`).
+ WillReturnRows(sqlmock.NewRows(teamDelCandCols).
+ AddRow(teamID, time.Now().UTC().Add(-31*24*time.Hour)))
+ mock.ExpectExec(`UPDATE teams\s+SET status = 'deletion_pending'`).
+ WithArgs(teamID).WillReturnError(errors.New("mark boom"))
+ // processTeam error → emitDeletionFailed audit row.
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+
+ w := NewTeamDeletionExecutorWorker(db, nil, nil, nil, "")
+ // Work logs the per-team error and continues; it returns nil overall.
+ _ = w.Work(context.Background(), localJob[TeamDeletionExecutorArgs]())
+}
+
+func TestTeamDeletion_FetchResources_ScanError(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ teamID := uuid.New()
+ seedTeamDeletionCandidate(mock, teamID)
+ // id column non-UUID → fetchTeamResources Scan errors → processTeam
+ // returns "fetch resources" error.
+ mock.ExpectQuery(`FROM resources\s+WHERE team_id`).WithArgs(teamID).
+ WillReturnRows(sqlmock.NewRows(teamDelResCols).
+ AddRow("not-a-uuid", "tok", "postgres", ""))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+
+ w := NewTeamDeletionExecutorWorker(db, nil, nil, nil, "")
+ _ = w.Work(context.Background(), localJob[TeamDeletionExecutorArgs]())
+}
+
+func TestTeamDeletion_DeleteS3Backups_ListError_InProcessTeam(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ teamID := uuid.New()
+ seedTeamDeletionCandidate(mock, teamID)
+ mock.ExpectQuery(`FROM resources\s+WHERE team_id`).WithArgs(teamID).
+ WillReturnRows(sqlmock.NewRows(teamDelResCols).
+ AddRow(uuid.New(), uuid.New().String(), "postgres", ""))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+
+ // s3 list yields an error → deleteS3BackupsForToken returns it →
+ // processTeam aborts with "delete s3 backups" error.
+ s3 := &fakeS3Deleter{listErr: errors.New("list boom")}
+ w := NewTeamDeletionExecutorWorker(db, nil, s3, nil, "instant-shared")
+ _ = w.Work(context.Background(), localJob[TeamDeletionExecutorArgs]())
+}
+
+func TestTeamDeletion_BeginTxError(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ teamID := uuid.New()
+ seedTeamDeletionCandidate(mock, teamID)
+ mock.ExpectQuery(`FROM resources\s+WHERE team_id`).WithArgs(teamID).
+ WillReturnRows(sqlmock.NewRows(teamDelResCols)) // no resources
+ mock.ExpectBegin().WillReturnError(errors.New("begin boom"))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+
+ w := NewTeamDeletionExecutorWorker(db, nil, nil, nil, "")
+ _ = w.Work(context.Background(), localJob[TeamDeletionExecutorArgs]())
+}
+
+func TestTeamDeletion_TxResourcePIIError(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ teamID := uuid.New()
+ seedTeamDeletionCandidate(mock, teamID)
+ mock.ExpectQuery(`FROM resources\s+WHERE team_id`).WithArgs(teamID).
+ WillReturnRows(sqlmock.NewRows(teamDelResCols))
+ mock.ExpectBegin()
+ mock.ExpectExec(`UPDATE resources\s+SET connection_url`).WithArgs(teamID).
+ WillReturnError(errors.New("null pii boom"))
+ mock.ExpectRollback()
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+
+ w := NewTeamDeletionExecutorWorker(db, nil, nil, nil, "")
+ _ = w.Work(context.Background(), localJob[TeamDeletionExecutorArgs]())
+}
+
+func TestTeamDeletion_FetchAppIDs_ScanError(t *testing.T) {
+ // k8s deleter wired → fetchTeamDeployAppIDs runs; a non-string app_id
+ // scan target won't fail (it's text), so force a query error instead
+ // to hit the fetch-deploy-app-ids error return.
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ teamID := uuid.New()
+ seedTeamDeletionCandidate(mock, teamID)
+ mock.ExpectQuery(`FROM resources\s+WHERE team_id`).WithArgs(teamID).
+ WillReturnRows(sqlmock.NewRows(teamDelResCols))
+ mock.ExpectQuery(`SELECT DISTINCT app_id\s+FROM deployments\s+WHERE team_id`).
+ WithArgs(teamID).WillReturnError(errors.New("appids boom"))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+
+ w := NewTeamDeletionExecutorWorker(db, nil, nil, &localNSDeleter{}, "")
+ _ = w.Work(context.Background(), localJob[TeamDeletionExecutorArgs]())
+}
+
+// localNSDeleter is a no-op K8sNamespaceDeleter for wiring the k8s step.
+type localNSDeleter struct{ delErr error }
+
+func (n localNSDeleter) DeleteNamespace(context.Context, string) error { return n.delErr }
+func (localNSDeleter) NamespaceExists(context.Context, string) (bool, error) {
+ return false, nil
+}
+
+func TestTeamDeletion_FetchResources_QueryError(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ teamID := uuid.New()
+ seedTeamDeletionCandidate(mock, teamID)
+ mock.ExpectQuery(`FROM resources\s+WHERE team_id`).WithArgs(teamID).
+ WillReturnError(errors.New("fetch resources boom"))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ w := NewTeamDeletionExecutorWorker(db, nil, nil, nil, "")
+ _ = w.Work(context.Background(), localJob[TeamDeletionExecutorArgs]())
+}
+
+func TestTeamDeletion_TxUserPIIError(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ teamID := uuid.New()
+ seedTeamDeletionCandidate(mock, teamID)
+ mock.ExpectQuery(`FROM resources\s+WHERE team_id`).WithArgs(teamID).
+ WillReturnRows(sqlmock.NewRows(teamDelResCols))
+ mock.ExpectBegin()
+ mock.ExpectExec(`UPDATE resources\s+SET connection_url`).WithArgs(teamID).
+ WillReturnResult(sqlmock.NewResult(0, 1))
+ mock.ExpectExec(`UPDATE users\s+SET email`).WithArgs(teamID).
+ WillReturnError(errors.New("user pii boom"))
+ mock.ExpectRollback()
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ w := NewTeamDeletionExecutorWorker(db, nil, nil, nil, "")
+ _ = w.Work(context.Background(), localJob[TeamDeletionExecutorArgs]())
+}
+
+func TestTeamDeletion_TxTeamStatusError(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ teamID := uuid.New()
+ seedTeamDeletionCandidate(mock, teamID)
+ mock.ExpectQuery(`FROM resources\s+WHERE team_id`).WithArgs(teamID).
+ WillReturnRows(sqlmock.NewRows(teamDelResCols))
+ mock.ExpectBegin()
+ mock.ExpectExec(`UPDATE resources\s+SET connection_url`).WithArgs(teamID).
+ WillReturnResult(sqlmock.NewResult(0, 1))
+ mock.ExpectExec(`UPDATE users\s+SET email`).WithArgs(teamID).
+ WillReturnResult(sqlmock.NewResult(0, 1))
+ mock.ExpectExec(`UPDATE teams\s+SET status\s+= 'tombstoned'`).WithArgs(teamID).
+ WillReturnError(errors.New("flip status boom"))
+ mock.ExpectRollback()
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ w := NewTeamDeletionExecutorWorker(db, nil, nil, nil, "")
+ _ = w.Work(context.Background(), localJob[TeamDeletionExecutorArgs]())
+}
+
+func TestTeamDeletion_S3DeleteSuccess_AccumulatesBytes(t *testing.T) {
+ // s3 wired + a resource with a real token + a list that yields one
+ // object that removes cleanly → deleteS3BackupsForToken returns
+ // (bytesFreed, nil) and processTeam accumulates s3BytesFreed (369).
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ teamID := uuid.New()
+ seedTeamDeletionCandidate(mock, teamID)
+ mock.ExpectQuery(`FROM resources\s+WHERE team_id`).WithArgs(teamID).
+ WillReturnRows(sqlmock.NewRows(teamDelResCols).
+ AddRow(uuid.New(), uuid.New().String(), "postgres", ""))
+ mock.ExpectBegin()
+ mock.ExpectExec(`UPDATE resources\s+SET connection_url`).WithArgs(teamID).
+ WillReturnResult(sqlmock.NewResult(0, 1))
+ mock.ExpectExec(`UPDATE users\s+SET email`).WithArgs(teamID).
+ WillReturnResult(sqlmock.NewResult(0, 1))
+ mock.ExpectExec(`UPDATE teams\s+SET status\s+= 'tombstoned'`).WithArgs(teamID).
+ WillReturnResult(sqlmock.NewResult(0, 1))
+ mock.ExpectCommit()
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+
+ s3 := &fakeS3Deleter{listObjects: []minio.ObjectInfo{{Key: "backups/tok/a.dump.gz", Size: 4096}}}
+ w := NewTeamDeletionExecutorWorker(db, nil, s3, nil, "instant-shared")
+ _ = w.Work(context.Background(), localJob[TeamDeletionExecutorArgs]())
+}
+
+func TestTeamDeletion_FetchAppIDs_RowScanError(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ teamID := uuid.New()
+ seedTeamDeletionCandidate(mock, teamID)
+ mock.ExpectQuery(`FROM resources\s+WHERE team_id`).WithArgs(teamID).
+ WillReturnRows(sqlmock.NewRows(teamDelResCols))
+ mock.ExpectQuery(`SELECT DISTINCT app_id\s+FROM deployments\s+WHERE team_id`).
+ WithArgs(teamID).
+ WillReturnRows(sqlmock.NewRows([]string{"app_id"}).
+ AddRow("app-1").RowError(0, errors.New("appid scan boom")))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ w := NewTeamDeletionExecutorWorker(db, nil, nil, localNSDeleter{}, "")
+ _ = w.Work(context.Background(), localJob[TeamDeletionExecutorArgs]())
+}
+
+func TestTeamDeletion_FetchResources_RowScanError(t *testing.T) {
+ // A RowError on the resources rowset surfaces as a Scan error inside
+ // fetchTeamResources' row loop (the `return nil, err` at 533).
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ teamID := uuid.New()
+ seedTeamDeletionCandidate(mock, teamID)
+ mock.ExpectQuery(`FROM resources\s+WHERE team_id`).WithArgs(teamID).
+ WillReturnRows(sqlmock.NewRows(teamDelResCols).
+ AddRow(uuid.New(), uuid.New().String(), "postgres", "").
+ RowError(0, errors.New("res scan boom")))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ w := NewTeamDeletionExecutorWorker(db, nil, nil, nil, "")
+ _ = w.Work(context.Background(), localJob[TeamDeletionExecutorArgs]())
+}
+
+func TestTeamDeletion_ProvisionerStep_SkipUnspecifiedAndDeprovisionError(t *testing.T) {
+ // A real (but unreachable) provisioner.Client exercises Step 2:
+ // - a "storage" resource maps to RESOURCE_TYPE_UNSPECIFIED → the
+ // skip-unspecified continue branch (379-391),
+ // - a "postgres" resource triggers a DeprovisionResource gRPC call
+ // that fails against the dead address → the deprovision-error
+ // return branch (393-397).
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ teamID := uuid.New()
+ seedTeamDeletionCandidate(mock, teamID)
+ mock.ExpectQuery(`FROM resources\s+WHERE team_id`).WithArgs(teamID).
+ WillReturnRows(sqlmock.NewRows(teamDelResCols).
+ AddRow(uuid.New(), uuid.New().String(), "storage", "").
+ AddRow(uuid.New(), uuid.New().String(), "postgres", "pr-1"))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+
+ // Dial a closed/unroutable address; the gRPC client is lazy so
+ // construction succeeds and DeprovisionResource fails at call time.
+ provCli, conn, derr := provisioner.NewClient("127.0.0.1:1", "secret")
+ if derr != nil {
+ t.Fatalf("provisioner.NewClient: %v", derr)
+ }
+ defer conn.Close()
+
+ w := NewTeamDeletionExecutorWorker(db, provCli, nil, nil, "")
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ _ = w.Work(ctx, localJob[TeamDeletionExecutorArgs]())
+}
+
+func TestTeamDeletion_NamespaceDeleteError(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ teamID := uuid.New()
+ seedTeamDeletionCandidate(mock, teamID)
+ mock.ExpectQuery(`FROM resources\s+WHERE team_id`).WithArgs(teamID).
+ WillReturnRows(sqlmock.NewRows(teamDelResCols))
+ mock.ExpectQuery(`SELECT DISTINCT app_id\s+FROM deployments\s+WHERE team_id`).
+ WithArgs(teamID).
+ WillReturnRows(sqlmock.NewRows([]string{"app_id"}).AddRow("app-123"))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ // DeleteNamespace errors → processTeam aborts before the tombstone tx.
+ w := NewTeamDeletionExecutorWorker(db, nil, nil, localNSDeleter{delErr: errors.New("ns boom")}, "")
+ _ = w.Work(context.Background(), localJob[TeamDeletionExecutorArgs]())
+}
+
+// ────────────────────────────────────────────────────────────────────
+// uptime_prober.go — defaultProvisionerDialer success + error
+// ────────────────────────────────────────────────────────────────────
+
+func TestUptime_DefaultProvisionerDialer_SuccessAndError(t *testing.T) {
+ // Success: dial a live httptest listener (TCP handshake completes).
+ srv := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
+ defer srv.Close()
+ addr := strings.TrimPrefix(srv.URL, "http://")
+ if err := defaultProvisionerDialer(context.Background(), addr); err != nil {
+ t.Fatalf("dial live listener should succeed, got %v", err)
+ }
+ // Error: dial an unroutable host.
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+ defer cancel()
+ if err := defaultProvisionerDialer(ctx, "192.0.2.1:50051"); err == nil {
+ t.Fatal("dial to unroutable host should error")
+ }
+}
+
+// ────────────────────────────────────────────────────────────────────
+// platform_db_backup.go — lock errors, dump-goroutine panic, default Now
+// ────────────────────────────────────────────────────────────────────
+
+func TestPlatformDBBackup_DefaultNowClosure_Returns(t *testing.T) {
+ // NewPlatformDBBackupWorker with Now=nil installs a UTC time.Now
+ // closure; invoking it exercises the closure body (line 238).
+ w := NewPlatformDBBackupWorker(PlatformDBBackupConfig{
+ DatabaseURL: "postgres://x@y/z", Bucket: "b", InnerPrefix: "platform-backups/",
+ })
+ if got := w.now(); got.IsZero() || got.Location() != time.UTC {
+ t.Errorf("default Now closure should return non-zero UTC time, got %v", got)
+ }
+}
+
+func TestPlatformDBBackup_LockQueryError(t *testing.T) {
+ // pg_try_advisory_lock query error → Work returns the wrapped error.
+ db, mock, err := sqlmock.New(
+ sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp),
+ sqlmock.MonitorPingsOption(false),
+ )
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ mock.ExpectQuery(`SELECT pg_try_advisory_lock`).WillReturnError(errors.New("lock query boom"))
+
+ w := newTestWorker(t, mock, db, &fakePgDumper{payload: []byte("x")}, newFakeS3(),
+ time.Date(2026, 5, 13, 2, 0, 0, 0, time.UTC))
+ if err := w.Work(context.Background(), fakePlatformBackupJob()); err == nil ||
+ !strings.Contains(err.Error(), "pg_try_advisory_lock") {
+ t.Fatalf("expected lock-query error, got %v", err)
+ }
+}
+
+func TestPlatformDBBackup_LockReleaseError_Logged(t *testing.T) {
+ // The deferred pg_advisory_unlock errors — logged, does not change the
+ // (otherwise successful) Work outcome.
+ db, mock, err := sqlmock.New(
+ sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp),
+ sqlmock.MonitorPingsOption(false),
+ )
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ expectAdvisoryLockAcquired(mock)
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1)) // started
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1)) // succeeded
+ mock.ExpectExec(`pg_advisory_unlock`).WillReturnError(errors.New("release boom"))
+
+ w := newTestWorker(t, mock, db, &fakePgDumper{payload: []byte("data")}, newFakeS3(),
+ time.Date(2026, 5, 13, 2, 0, 0, 0, time.UTC))
+ if err := w.Work(context.Background(), fakePlatformBackupJob()); err != nil {
+ t.Fatalf("Work should succeed despite unlock error, got %v", err)
+ }
+}
+
+func TestPlatformDBBackup_DumpGoroutinePanic_Recovered(t *testing.T) {
+ // A dumper that panics is caught by the dump-goroutine's recover()
+ // boundary; Work surfaces a non-nil error rather than crashing.
+ db, mock, err := sqlmock.New(
+ sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp),
+ sqlmock.MonitorPingsOption(false),
+ )
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ expectAdvisoryLockAcquired(mock)
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1)) // started
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1)) // failed
+ expectAdvisoryUnlock(mock)
+
+ w := newTestWorker(t, mock, db, nil, newFakeS3(),
+ time.Date(2026, 5, 13, 2, 0, 0, 0, time.UTC))
+ w.dumper = panicDumper{}
+ if err := w.Work(context.Background(), fakePlatformBackupJob()); err == nil {
+ t.Fatal("expected error from panicking dumper")
+ }
+}
+
+type panicDumper struct{}
+
+func (panicDumper) Dump(_ context.Context, _ string, _ io.Writer) (int64, error) {
+ panic("dump exploded")
+}
+
+// ────────────────────────────────────────────────────────────────────
+// customer_backup_runner.go — scan_failed + rows.Err + retention scan
+// ────────────────────────────────────────────────────────────────────
+
+func TestRunner_Work_ScanError_SkipsRow(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'pending'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT b.id::text`).WithArgs(backupBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "tier_at_backup", "backup_kind",
+ "token", "connection_url", "resource_type", "team_id",
+ }).AddRow("bk", "res", "pro", "scheduled", "tk", "url", "postgres", "not-a-uuid"))
+ mock.ExpectQuery(`SELECT id::text, s3_key`).WillReturnRows(sqlmock.NewRows([]string{"id", "s3_key"}))
+
+ w := &CustomerBackupRunnerWorker{
+ db: db, store: newFakeBackupStore(), pgDump: &fakePgDump{},
+ bucket: "b", prefix: "p", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: backupBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRunnerJob()); err != nil {
+ t.Fatalf("Work should skip unscannable row, got %v", err)
+ }
+}
+
+func TestRunner_Work_RowsError_ReturnsError(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ mock.ExpectExec(`UPDATE resource_backups\s+SET status = 'pending'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ rows := sqlmock.NewRows([]string{
+ "id", "resource_id", "tier_at_backup", "backup_kind",
+ "token", "connection_url", "resource_type", "team_id",
+ }).AddRow("bk", "res", "pro", "scheduled", "tk", "url", "postgres", uuid.New()).
+ RowError(0, errors.New("rows boom"))
+ mock.ExpectQuery(`SELECT b.id::text`).WithArgs(backupBatchSize).WillReturnRows(rows)
+
+ w := &CustomerBackupRunnerWorker{
+ db: db, store: newFakeBackupStore(), pgDump: &fakePgDump{},
+ bucket: "b", prefix: "p", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: backupBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRunnerJob()); err == nil ||
+ !strings.Contains(err.Error(), "rows error") {
+ t.Fatalf("expected rows-error, got %v", err)
+ }
+}
+
+func TestRunner_RetentionSweep_ScanError_SkipsVictim(t *testing.T) {
+ // runRetentionSweep selects (id, s3_key) per tier; a row-level scan
+ // error hits the retention_scan_failed continue branch.
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ // First tier query returns a row whose s3_key column holds a non-string
+ // value the *string scan target rejects → a real per-row Scan error →
+ // the retention_scan_failed continue branch (not rows.Err).
+ mock.ExpectQuery(`SELECT id::text, s3_key`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "s3_key"}).
+ AddRow("bk1", []byte{0xff, 0xfe}).AddRow(nil, nil))
+ // Remaining tier queries return empty (the sweep iterates every tier).
+ for i := 0; i < 12; i++ {
+ mock.ExpectQuery(`SELECT id::text, s3_key`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "s3_key"}))
+ }
+
+ w := &CustomerBackupRunnerWorker{
+ db: db, store: newFakeBackupStore(),
+ bucket: "b", prefix: "p", now: time.Now,
+ }
+ // runRetentionSweep is unexported; call directly (same package).
+ w.runRetentionSweep(context.Background())
+}
+
+// ────────────────────────────────────────────────────────────────────
+// customer_restore_runner.go — decrypt error + S3 read error branches
+// ────────────────────────────────────────────────────────────────────
+
+func TestRestoreRunner_ProcessRestore_DecryptError_MarksFailed(t *testing.T) {
+ // A connection_url that is non-empty but not valid ciphertext for the
+ // configured AES key → crypto.Decrypt errors → markRestoreFailed.
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatalf("sqlmock.New: %v", err)
+ }
+ defer db.Close()
+ restoreID := "rrrrrrr0-1111-2222-3333-444444444444"
+ resID := "22222222-2222-2222-2222-222222222222"
+ teamID := uuid.New()
+ mock.ExpectExec(`UPDATE resource_restores\s+SET status\s+= 'failed'`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT rr\.id::text`).WithArgs(restoreBatchSize).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "resource_id", "backup_id", "s3_key", "sha256",
+ "connection_url", "resource_type", "token", "team_id",
+ }).AddRow(restoreID, resID, "bk", "k", nil, "GARBAGE-CIPHERTEXT", "postgres", "tk", teamID))
+ mock.ExpectQuery(`UPDATE resource_restores\s+SET status = 'running'`).
+ WithArgs(restoreID).WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(restoreID))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`UPDATE resource_restores\s+SET status = 'failed'`).
+ WithArgs(restoreID, sqlmock.AnyArg()).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1))
+
+ w := &CustomerRestoreRunnerWorker{
+ db: db, store: newFakeBackupStore(), pgRestore: &fakePgRestore{},
+ bucket: "instant-shared", aesKey: testAESKeyHex,
+ now: time.Now, timeout: time.Minute, batchN: restoreBatchSize,
+ }
+ if err := w.Work(context.Background(), fakeRestoreJob()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+// ────────────────────────────────────────────────────────────────────
+// geodb.go — Work happy path: download → extract → rename
+// ────────────────────────────────────────────────────────────────────
+
+func TestGeoDB_Work_HappyPath_DownloadExtractRename(t *testing.T) {
+ // Build a real gzipped tarball holding a single *.mmdb member.
+ var buf bytes.Buffer
+ gz := gzip.NewWriter(&buf)
+ tw := tar.NewWriter(gz)
+ body := []byte("real-mmdb-bytes")
+ hdr := &tar.Header{
+ Name: "GeoLite2-City_20260522/GeoLite2-City.mmdb",
+ Typeflag: tar.TypeReg,
+ Size: int64(len(body)),
+ Mode: 0o644,
+ }
+ if err := tw.WriteHeader(hdr); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := tw.Write(body); err != nil {
+ t.Fatal(err)
+ }
+ _ = tw.Close()
+ _ = gz.Close()
+ tarball := buf.Bytes()
+
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write(tarball)
+ }))
+ defer srv.Close()
+
+ // Point the download template at the httptest server. Restore the
+ // production value afterwards so other tests are unaffected.
+ orig := geoLite2DownloadURL
+ geoLite2DownloadURL = srv.URL + "?license_key=%s"
+ defer func() { geoLite2DownloadURL = orig }()
+
+ dst := filepath.Join(t.TempDir(), "GeoLite2-City.mmdb")
+ w := NewRefreshGeoDBWorker()
+ job := &river.Job[RefreshGeoDBArgs]{
+ Args: RefreshGeoDBArgs{LicenseKey: "fake-key", DBPath: dst},
+ JobRow: &rivertype.JobRow{ID: 1},
+ }
+ if err := w.Work(context.Background(), job); err != nil {
+ t.Fatalf("Work happy path: %v", err)
+ }
+ got, err := os.ReadFile(dst)
+ if err != nil {
+ t.Fatalf("read extracted mmdb: %v", err)
+ }
+ if !bytes.Equal(got, body) {
+ t.Errorf("extracted bytes mismatch: %q vs %q", got, body)
+ }
+ // The fetch marker should have been stamped.
+ if _, err := os.Stat(dst + geoLite2FetchMarkerSuffix); err != nil {
+ t.Errorf("fetch marker not stamped: %v", err)
+ }
+}
+
+func TestGeoDB_Extract_SkipsNonMMDBThenFinds(t *testing.T) {
+ // A tarball whose first member is a non-.mmdb regular file (hits the
+ // `continue` skip branch) followed by the real .mmdb member.
+ var buf bytes.Buffer
+ gz := gzip.NewWriter(&buf)
+ tw := tar.NewWriter(gz)
+ for _, m := range []struct {
+ name string
+ body []byte
+ }{
+ {"GeoLite2-City_20260522/COPYRIGHT.txt", []byte("copyright")},
+ {"GeoLite2-City_20260522/GeoLite2-City.mmdb", []byte("the-db")},
+ } {
+ if err := tw.WriteHeader(&tar.Header{Name: m.name, Typeflag: tar.TypeReg, Size: int64(len(m.body)), Mode: 0o644}); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := tw.Write(m.body); err != nil {
+ t.Fatal(err)
+ }
+ }
+ _ = tw.Close()
+ _ = gz.Close()
+
+ dst := filepath.Join(t.TempDir(), "out.mmdb")
+ if err := extractGeoLite2MMDB(&buf, dst); err != nil {
+ t.Fatalf("extract should skip txt and find mmdb: %v", err)
+ }
+ got, _ := os.ReadFile(dst)
+ if !bytes.Equal(got, []byte("the-db")) {
+ t.Errorf("wrong member extracted: %q", got)
+ }
+}
+
+func TestGeoDB_Extract_CreateTempFileError(t *testing.T) {
+ // dstPath points into a directory that does not exist → os.Create fails
+ // → the "create temp file" error branch fires.
+ var buf bytes.Buffer
+ gz := gzip.NewWriter(&buf)
+ tw := tar.NewWriter(gz)
+ body := []byte("db")
+ _ = tw.WriteHeader(&tar.Header{Name: "x/GeoLite2-City.mmdb", Typeflag: tar.TypeReg, Size: int64(len(body)), Mode: 0o644})
+ _, _ = tw.Write(body)
+ _ = tw.Close()
+ _ = gz.Close()
+
+ dst := filepath.Join(t.TempDir(), "no-such-dir", "out.mmdb")
+ err := extractGeoLite2MMDB(&buf, dst)
+ if err == nil || !strings.Contains(err.Error(), "create temp file") {
+ t.Fatalf("expected create-temp-file error, got %v", err)
+ }
+}
+
+func TestGeoDB_Work_RenameError(t *testing.T) {
+ // A valid tarball downloads + extracts, but the destination path is a
+ // directory, so os.Rename(tmp, dir) fails → Work returns the wrapped
+ // "rename failed" error and removes the tmp file.
+ var buf bytes.Buffer
+ gz := gzip.NewWriter(&buf)
+ tw := tar.NewWriter(gz)
+ body := []byte("db-bytes")
+ _ = tw.WriteHeader(&tar.Header{Name: "d/GeoLite2-City.mmdb", Typeflag: tar.TypeReg, Size: int64(len(body)), Mode: 0o644})
+ _, _ = tw.Write(body)
+ _ = tw.Close()
+ _ = gz.Close()
+ tarball := buf.Bytes()
+
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write(tarball)
+ }))
+ defer srv.Close()
+ orig := geoLite2DownloadURL
+ geoLite2DownloadURL = srv.URL + "?license_key=%s"
+ defer func() { geoLite2DownloadURL = orig }()
+
+ // DBPath is an existing directory → the rename of .tmp onto it
+ // fails because you cannot rename a file over a non-empty directory.
+ dstDir := t.TempDir()
+ // Make the dir non-empty so the rename is guaranteed to fail.
+ if err := os.WriteFile(filepath.Join(dstDir, "keep"), []byte("x"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ w := NewRefreshGeoDBWorker()
+ job := &river.Job[RefreshGeoDBArgs]{
+ Args: RefreshGeoDBArgs{LicenseKey: "fake-key", DBPath: dstDir},
+ JobRow: &rivertype.JobRow{ID: 1},
+ }
+ err := w.Work(context.Background(), job)
+ if err == nil || !strings.Contains(err.Error(), "rename failed") {
+ t.Fatalf("expected rename-failed error, got %v", err)
+ }
+ if _, statErr := os.Stat(dstDir + ".tmp"); !os.IsNotExist(statErr) {
+ t.Errorf("tmp file should be removed on rename failure")
+ }
+}
+
+func TestGeoDB_Work_BuildRequestError(t *testing.T) {
+ // A license key containing a control character makes the interpolated
+ // URL invalid → http.NewRequestWithContext fails (the build-request
+ // branch), before any network dial.
+ orig := geoLite2DownloadURL
+ geoLite2DownloadURL = "http://example.com/db?key=%s\x7f"
+ defer func() { geoLite2DownloadURL = orig }()
+
+ dst := filepath.Join(t.TempDir(), "GeoLite2-City.mmdb")
+ w := NewRefreshGeoDBWorker()
+ job := &river.Job[RefreshGeoDBArgs]{
+ Args: RefreshGeoDBArgs{LicenseKey: "k", DBPath: dst},
+ JobRow: &rivertype.JobRow{ID: 1},
+ }
+ if err := w.Work(context.Background(), job); err == nil ||
+ !strings.Contains(err.Error(), "build request") {
+ t.Fatalf("expected build-request error, got %v", err)
+ }
+}
+
+func TestGeoDB_Extract_TruncatedTarBody_Errors(t *testing.T) {
+ // A tar header declaring Size=100 but with no body bytes makes the
+ // bounded io.Copy hit an unexpected EOF → the "write temp file" /
+ // "read tar entry" error branch.
+ var buf bytes.Buffer
+ gz := gzip.NewWriter(&buf)
+ tw := tar.NewWriter(gz)
+ // Write a header whose declared size exceeds the body we actually
+ // flush; then close the gzip stream early to truncate the tar.
+ _ = tw.WriteHeader(&tar.Header{Name: "d/GeoLite2-City.mmdb", Typeflag: tar.TypeReg, Size: 100, Mode: 0o644})
+ _, _ = tw.Write([]byte("short")) // only 5 of 100 bytes
+ // Intentionally do NOT call tw.Close() (which would error on the size
+ // mismatch); flush gzip directly to produce a truncated archive.
+ _ = gz.Close()
+
+ dst := filepath.Join(t.TempDir(), "out.mmdb")
+ if err := extractGeoLite2MMDB(&buf, dst); err == nil {
+ t.Fatal("expected error from truncated tar body")
+ }
+}
+
+func TestGeoDB_Work_Non200Status_ReturnsError(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusForbidden)
+ }))
+ defer srv.Close()
+ orig := geoLite2DownloadURL
+ geoLite2DownloadURL = srv.URL + "?license_key=%s"
+ defer func() { geoLite2DownloadURL = orig }()
+
+ dst := filepath.Join(t.TempDir(), "GeoLite2-City.mmdb")
+ w := NewRefreshGeoDBWorker()
+ job := &river.Job[RefreshGeoDBArgs]{
+ Args: RefreshGeoDBArgs{LicenseKey: "fake-key", DBPath: dst},
+ JobRow: &rivertype.JobRow{ID: 1},
+ }
+ err := w.Work(context.Background(), job)
+ if err == nil || !strings.Contains(err.Error(), "unexpected status") {
+ t.Fatalf("expected unexpected-status error, got %v", err)
+ }
+}
+
+func TestGeoDB_Work_BadTarball_ExtractFails(t *testing.T) {
+ // 200 OK but the body is not a valid gzip tarball → extract fails →
+ // Work returns the wrapped "extract failed" error and removes the tmp.
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte("not-a-gzip-tarball"))
+ }))
+ defer srv.Close()
+ orig := geoLite2DownloadURL
+ geoLite2DownloadURL = srv.URL + "?license_key=%s"
+ defer func() { geoLite2DownloadURL = orig }()
+
+ dst := filepath.Join(t.TempDir(), "GeoLite2-City.mmdb")
+ w := NewRefreshGeoDBWorker()
+ job := &river.Job[RefreshGeoDBArgs]{
+ Args: RefreshGeoDBArgs{LicenseKey: "fake-key", DBPath: dst},
+ JobRow: &rivertype.JobRow{ID: 1},
+ }
+ err := w.Work(context.Background(), job)
+ if err == nil || !strings.Contains(err.Error(), "extract failed") {
+ t.Fatalf("expected extract-failed error, got %v", err)
+ }
+ // The tmp file must have been cleaned up.
+ if _, statErr := os.Stat(dst + ".tmp"); !os.IsNotExist(statErr) {
+ t.Errorf("tmp file should be removed on extract failure")
+ }
+ _ = fmt.Sprint(err)
+}
diff --git a/internal/jobs/coverage_misc_test.go b/internal/jobs/coverage_misc_test.go
new file mode 100644
index 0000000..3b759f3
--- /dev/null
+++ b/internal/jobs/coverage_misc_test.go
@@ -0,0 +1,2253 @@
+package jobs
+
+// coverage_misc_test.go — drives ≥95% coverage on the worker job files
+// targeted by the misc-job coverage worktree:
+// propagation_runner.go, provisioner_reconciler.go, resource_heartbeat.go,
+// prober.go, real_prober.go, uptime_prober.go, geodb.go,
+// team_deletion_executor.go, team_deletion_audit_kinds.go,
+// team_deletion_s3_adapter.go, chaos_lease_recovery.go.
+//
+// All tests intentionally use names matching the brief's filter set:
+// TestPropagation*|TestProvisionerReconcile*|TestHeartbeat*|TestProber*|
+// TestGeoDB*|TestTeamDeletion*|TestAuditKind*|TestS3Adapter*|TestChaos*
+
+import (
+ "archive/tar"
+ "bytes"
+ "compress/gzip"
+ "context"
+ "database/sql"
+ "errors"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ sqlmock "github.com/DATA-DOG/go-sqlmock"
+ "github.com/google/uuid"
+ minio "github.com/minio/minio-go/v7"
+ "github.com/redis/go-redis/v9"
+ "github.com/riverqueue/river"
+ "github.com/riverqueue/river/rivertype"
+
+ commonv1 "instant.dev/proto/common/v1"
+ "instant.dev/worker/internal/config"
+)
+
+// localJob returns a minimal *river.Job for in-package callers.
+func localJob[T river.JobArgs]() *river.Job[T] {
+ return &river.Job[T]{JobRow: &rivertype.JobRow{ID: 42}}
+}
+
+// localFakeProber is a tiny ResourceProber double for in-package tests.
+type localFakeProber struct {
+ mu sync.Mutex
+ outcome ProbeOutcome
+ err error
+ calls int
+}
+
+func (p *localFakeProber) Probe(_ context.Context, _, _ string) (ProbeOutcome, error) {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ p.calls++
+ return p.outcome, p.err
+}
+
+// ─── prober.go: NoopProber dispatch ────────────────────────────────────────────
+
+func TestProber_Noop_AllResourceTypesReachable(t *testing.T) {
+ // Hit Probe(...) explicitly for both arms (resourceType ignored). Pins
+ // the 100% surface on prober.go's only function.
+ for _, rt := range []string{"postgres", "redis", "mongodb", "queue", "storage", "webhook", ""} {
+ out, err := NoopProber{}.Probe(context.Background(), rt, "anything")
+ if out != ProbeReachable || err != nil {
+ t.Errorf("NoopProber{%q} = (%v,%v), want (ProbeReachable,nil)", rt, out, err)
+ }
+ }
+}
+
+func TestProber_ErrUnconfigured_Sentinel(t *testing.T) {
+ // Sentinel must be a stable, non-nil error for callers to check via ==.
+ if ErrProberUnconfigured == nil {
+ t.Fatal("ErrProberUnconfigured must be non-nil")
+ }
+ if !strings.Contains(ErrProberUnconfigured.Error(), "not configured") {
+ t.Errorf("ErrProberUnconfigured text = %q; expected to contain 'not configured'", ErrProberUnconfigured.Error())
+ }
+}
+
+// ─── resource_heartbeat.go ────────────────────────────────────────────────────
+
+func TestHeartbeat_Kind_ReturnsConstant(t *testing.T) {
+ if k := (ResourceHeartbeatArgs{}).Kind(); k != "resource_heartbeat" {
+ t.Errorf("ResourceHeartbeatArgs.Kind = %q, want resource_heartbeat", k)
+ }
+}
+
+func TestHeartbeat_PeriodicInterval_ProdVsDev(t *testing.T) {
+ if got := resourceHeartbeatPeriodicInterval("production"); got != resourceHeartbeatInterval {
+ t.Errorf("production interval = %s, want %s", got, resourceHeartbeatInterval)
+ }
+ if got := resourceHeartbeatPeriodicInterval("development"); got != resourceHeartbeatDevInterval {
+ t.Errorf("dev interval = %s, want %s", got, resourceHeartbeatDevInterval)
+ }
+ if got := resourceHeartbeatPeriodicInterval(""); got != resourceHeartbeatDevInterval {
+ t.Errorf("empty env interval = %s, want dev %s", got, resourceHeartbeatDevInterval)
+ }
+}
+
+func TestHeartbeat_NilProberFallsBackToNoop(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ mock.ExpectQuery(`FROM resources`).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "token", "resource_type", "connection_url",
+ "team_id_text", "degraded", "last_seen_at",
+ }))
+ w := NewResourceHeartbeatWorker(db, nil) // nil → NoopProber
+ if err := w.Work(context.Background(), localJob[ResourceHeartbeatArgs]()); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+ if err := mock.ExpectationsWereMet(); err != nil {
+ t.Errorf("unmet: %v", err)
+ }
+}
+
+func TestHeartbeat_FlapWindowSuppressesAuditRow(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ resID := uuid.New()
+ token := uuid.New()
+ teamID := uuid.New().String()
+
+ // wasDegraded=true + fresh last_seen_at + probe fails → markDegraded
+ // gated UPDATE returns 0 rows (within flap window) → no audit row.
+ mock.ExpectQuery(`FROM resources`).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "token", "resource_type", "connection_url",
+ "team_id_text", "degraded", "last_seen_at",
+ }).AddRow(resID, token, "redis", "url", teamID, true, sql.NullTime{Time: time.Now(), Valid: true}))
+ mock.ExpectExec(`UPDATE resources\s+SET degraded = true`).
+ WithArgs(resID, sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(0, 0)) // 0 rows = inside flap window
+ mock.ExpectQuery(`SELECT resource_type, COUNT\(\*\)`).
+ WillReturnRows(sqlmock.NewRows([]string{"resource_type", "count"}))
+
+ w := NewResourceHeartbeatWorker(db, &localFakeProber{outcome: ProbeUnreachable, err: errors.New("boom")})
+ if err := w.Work(context.Background(), localJob[ResourceHeartbeatArgs]()); err != nil {
+ t.Fatal(err)
+ }
+ if err := mock.ExpectationsWereMet(); err != nil {
+ t.Errorf("unmet: %v", err)
+ }
+}
+
+func TestHeartbeat_HealthyUpdateError_LoggedAndContinues(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ resID := uuid.New()
+ mock.ExpectQuery(`FROM resources`).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "token", "resource_type", "connection_url",
+ "team_id_text", "degraded", "last_seen_at",
+ }).AddRow(resID, uuid.New(), "postgres", "url", "", false, sql.NullTime{}))
+ mock.ExpectExec(`UPDATE resources\s+SET last_seen_at`).
+ WithArgs(resID).
+ WillReturnError(errors.New("update failed"))
+ mock.ExpectQuery(`SELECT resource_type, COUNT\(\*\)`).
+ WillReturnRows(sqlmock.NewRows([]string{"resource_type", "count"}))
+
+ w := NewResourceHeartbeatWorker(db, &localFakeProber{outcome: ProbeReachable})
+ if err := w.Work(context.Background(), localJob[ResourceHeartbeatArgs]()); err != nil {
+ t.Fatalf("fail-open: Work should return nil, got %v", err)
+ }
+ if err := mock.ExpectationsWereMet(); err != nil {
+ t.Errorf("unmet: %v", err)
+ }
+}
+
+func TestHeartbeat_DegradedUpdateError_LoggedAndContinues(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ resID := uuid.New()
+ mock.ExpectQuery(`FROM resources`).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "token", "resource_type", "connection_url",
+ "team_id_text", "degraded", "last_seen_at",
+ }).AddRow(resID, uuid.New(), "postgres", "url", "", false, sql.NullTime{}))
+ mock.ExpectExec(`UPDATE resources\s+SET degraded = true`).
+ WithArgs(resID, sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnError(errors.New("degrade update failed"))
+ mock.ExpectQuery(`SELECT resource_type, COUNT\(\*\)`).
+ WillReturnRows(sqlmock.NewRows([]string{"resource_type", "count"}))
+
+ w := NewResourceHeartbeatWorker(db, &localFakeProber{outcome: ProbeUnreachable, err: errors.New("boom")})
+ if err := w.Work(context.Background(), localJob[ResourceHeartbeatArgs]()); err != nil {
+ t.Fatalf("fail-open: %v", err)
+ }
+ if err := mock.ExpectationsWereMet(); err != nil {
+ t.Errorf("unmet: %v", err)
+ }
+}
+
+func TestHeartbeat_GaugeQueryError_DoesNotFailWork(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ mock.ExpectQuery(`FROM resources`).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "token", "resource_type", "connection_url",
+ "team_id_text", "degraded", "last_seen_at",
+ }).AddRow(uuid.New(), uuid.New(), "webhook", "", "", false, sql.NullTime{}))
+ mock.ExpectQuery(`SELECT resource_type, COUNT\(\*\)`).
+ WillReturnError(errors.New("gauge query failed"))
+
+ w := NewResourceHeartbeatWorker(db, &localFakeProber{outcome: ProbeSkip})
+ if err := w.Work(context.Background(), localJob[ResourceHeartbeatArgs]()); err != nil {
+ t.Fatalf("gauge error must not fail Work: %v", err)
+ }
+}
+
+// ─── provisioner_reconciler.go: refundQuota + helpers + Kind ──────────────────
+
+func TestProvisionerReconcile_Kind_ReturnsConstant(t *testing.T) {
+ if k := (ProvisionerReconcilerArgs{}).Kind(); k != "provisioner_reconciler" {
+ t.Errorf("Kind = %q", k)
+ }
+}
+
+func TestProvisionerReconcile_RefundQuota_NilRedisIsNoOp(t *testing.T) {
+ w := &ProvisionerReconcilerWorker{}
+ // nil rdb branch — must return cleanly.
+ w.refundQuota(context.Background(), reconcilerCandidate{
+ id: uuid.New(),
+ resourceType: "postgres",
+ teamID: sql.NullString{String: uuid.New().String(), Valid: true},
+ })
+}
+
+func TestProvisionerReconcile_RefundQuota_AnonymousSkipped(t *testing.T) {
+ // rdb non-nil but team_id NULL → log+skip branch.
+ rdb := redis.NewClient(&redis.Options{Addr: "127.0.0.1:1"}) // unreachable, but never called
+ defer rdb.Close()
+ w := &ProvisionerReconcilerWorker{rdb: rdb}
+ w.refundQuota(context.Background(), reconcilerCandidate{
+ id: uuid.New(),
+ resourceType: "redis",
+ teamID: sql.NullString{}, // invalid → anonymous path
+ })
+}
+
+func TestProvisionerReconcile_RefundQuota_DecrError_Logged(t *testing.T) {
+ // rdb pointed at an unreachable port → Decr returns an error → logged
+ // branch fires.
+ rdb := redis.NewClient(&redis.Options{
+ Addr: "127.0.0.1:1",
+ DialTimeout: 100 * time.Millisecond,
+ })
+ defer rdb.Close()
+ w := &ProvisionerReconcilerWorker{rdb: rdb}
+ ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
+ defer cancel()
+ w.refundQuota(ctx, reconcilerCandidate{
+ id: uuid.New(),
+ resourceType: "postgres",
+ teamID: sql.NullString{String: uuid.New().String(), Valid: true},
+ })
+}
+
+func TestProvisionerReconcile_NullableTeamID_EmptyAndValid(t *testing.T) {
+ if got := nullableTeamID(sql.NullString{}); got != nil {
+ t.Errorf("invalid NullString → %v, want nil", got)
+ }
+ if got := nullableTeamID(sql.NullString{Valid: true, String: ""}); got != nil {
+ t.Errorf("empty-string NullString → %v, want nil", got)
+ }
+ id := uuid.New().String()
+ if got := nullableTeamID(sql.NullString{Valid: true, String: id}); got != id {
+ t.Errorf("valid NullString → %v, want %s", got, id)
+ }
+}
+
+func TestProvisionerReconcile_ProbeErrString_NilGuards(t *testing.T) {
+ if s := probeErrString(nil); !strings.Contains(s, "no error message") {
+ t.Errorf("nil err sentinel text = %q", s)
+ }
+ if s := probeErrString(errors.New("real failure")); s != "real failure" {
+ t.Errorf("got %q", s)
+ }
+}
+
+func TestProvisionerReconcile_TruncateReason_HonorsCap(t *testing.T) {
+ if got := truncateReason("hi"); got != "hi" {
+ t.Errorf("short returned changed: %q", got)
+ }
+ long := strings.Repeat("x", 1000)
+ if got := truncateReason(long); len(got) != 500 {
+ t.Errorf("truncated len = %d, want 500", len(got))
+ }
+}
+
+func TestProvisionerReconcile_RecoveredAuditError_LoggedFailOpen(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ resID := uuid.New()
+ teamID := uuid.New().String()
+
+ mock.ExpectQuery(`FROM resources`).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "token", "resource_type", "connection_url", "team_id_text",
+ }).AddRow(resID, uuid.New(), "postgres", "url", teamID))
+ mock.ExpectExec(`UPDATE resources\s+SET status = 'active'`).
+ WithArgs(resID).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs(teamID, "system", "provisioner.reconcile_recovered", sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnError(errors.New("audit insert failed"))
+
+ w := NewProvisionerReconcilerWorker(db, nil, &localFakeProber{outcome: ProbeReachable})
+ if err := w.Work(context.Background(), localJob[ProvisionerReconcilerArgs]()); err != nil {
+ t.Fatalf("fail-open: %v", err)
+ }
+}
+
+func TestProvisionerReconcile_AbandonedAuditError_LoggedFailOpen(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ resID := uuid.New()
+ teamID := uuid.New().String()
+ mock.ExpectQuery(`FROM resources`).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "token", "resource_type", "connection_url", "team_id_text",
+ }).AddRow(resID, uuid.New(), "postgres", "url", teamID))
+ mock.ExpectExec(`UPDATE resources\s+SET status = 'failed'`).
+ WithArgs(resID).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs(teamID, "system", "provisioner.reconcile_abandoned", sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnError(errors.New("audit insert failed"))
+
+ w := NewProvisionerReconcilerWorker(db, nil, &localFakeProber{outcome: ProbeUnreachable, err: errors.New("dial refused")})
+ if err := w.Work(context.Background(), localJob[ProvisionerReconcilerArgs]()); err != nil {
+ t.Fatalf("fail-open: %v", err)
+ }
+}
+
+func TestProvisionerReconcile_SkipUpdateError_LoggedFailOpen(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ resID := uuid.New()
+ mock.ExpectQuery(`FROM resources`).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "token", "resource_type", "connection_url", "team_id_text",
+ }).AddRow(resID, uuid.New(), "webhook", "", ""))
+ mock.ExpectExec(`UPDATE resources SET last_reconciled_at`).
+ WithArgs(resID).
+ WillReturnError(errors.New("stamp failed"))
+
+ w := NewProvisionerReconcilerWorker(db, nil, &localFakeProber{outcome: ProbeSkip})
+ if err := w.Work(context.Background(), localJob[ProvisionerReconcilerArgs]()); err != nil {
+ t.Fatalf("fail-open: %v", err)
+ }
+}
+
+func TestProvisionerReconcile_RowScanFailure_SkipsRow(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ // Row with a malformed UUID in id → Scan fails → row skipped → Work
+ // returns nil with no candidates.
+ mock.ExpectQuery(`FROM resources`).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "token", "resource_type", "connection_url", "team_id_text",
+ }).AddRow("not-a-uuid", "also-not-uuid", "postgres", "url", ""))
+
+ w := NewProvisionerReconcilerWorker(db, nil, &localFakeProber{outcome: ProbeReachable})
+ if err := w.Work(context.Background(), localJob[ProvisionerReconcilerArgs]()); err != nil {
+ t.Fatalf("scan errors must not propagate: %v", err)
+ }
+}
+
+// ─── propagation_runner.go: Kind, Interval ─────────────────────────────────────
+
+func TestPropagation_Kind_ReturnsConstant(t *testing.T) {
+ if k := (PropagationRunnerArgs{}).Kind(); k != "propagation_runner" {
+ t.Errorf("Kind = %q", k)
+ }
+}
+
+func TestPropagation_Interval_DefaultsAndOverride(t *testing.T) {
+ prev := os.Getenv("PROPAGATION_RUNNER_INTERVAL")
+ defer os.Setenv("PROPAGATION_RUNNER_INTERVAL", prev)
+
+ os.Unsetenv("PROPAGATION_RUNNER_INTERVAL")
+ if got := PropagationRunnerInterval(); got != propagationDefaultInterval {
+ t.Errorf("unset = %s, want %s", got, propagationDefaultInterval)
+ }
+
+ os.Setenv("PROPAGATION_RUNNER_INTERVAL", "10s")
+ if got := PropagationRunnerInterval(); got != 10*time.Second {
+ t.Errorf("override = %s, want 10s", got)
+ }
+
+ os.Setenv("PROPAGATION_RUNNER_INTERVAL", "garbage")
+ if got := PropagationRunnerInterval(); got != propagationDefaultInterval {
+ t.Errorf("bad value → fallback, got %s", got)
+ }
+
+ os.Setenv("PROPAGATION_RUNNER_INTERVAL", "0s")
+ if got := PropagationRunnerInterval(); got != propagationDefaultInterval {
+ t.Errorf("non-positive value → fallback, got %s", got)
+ }
+
+ os.Setenv("PROPAGATION_RUNNER_INTERVAL", " ")
+ if got := PropagationRunnerInterval(); got != propagationDefaultInterval {
+ t.Errorf("whitespace-only value → fallback, got %s", got)
+ }
+}
+
+func TestPropagation_NilRegrader_LogsWarnAndReturns(t *testing.T) {
+ db, _, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ w := NewPropagationRunnerWorker(db, nil, nil)
+ if err := w.Work(context.Background(), localJob[PropagationRunnerArgs]()); err != nil {
+ t.Fatalf("Work with nil regrader must return nil: %v", err)
+ }
+}
+
+func TestPropagation_PickEligible_TopLevelSelectError(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ mock.ExpectBegin()
+ mock.ExpectQuery(`SELECT id, kind, team_id`).WillReturnError(errors.New("select failed"))
+ mock.ExpectRollback()
+ w := NewPropagationRunnerWorker(db, nil, &stubPropagationRegrader{outcome: regradeOutcome{Applied: true}})
+ if err := w.Work(context.Background(), localJob[PropagationRunnerArgs]()); err == nil {
+ t.Fatal("expected error from pick failure")
+ }
+}
+
+func TestPropagation_PgUUIDArray_FormatsAsArrayLiteral(t *testing.T) {
+ if got := pgUUIDArray(nil); got != "{}" {
+ t.Errorf("nil = %q, want {}", got)
+ }
+ if got := pgUUIDArray([]uuid.UUID{}); got != "{}" {
+ t.Errorf("empty = %q, want {}", got)
+ }
+ one := uuid.New()
+ if got := pgUUIDArray([]uuid.UUID{one}); got != "{"+one.String()+"}" {
+ t.Errorf("one element = %q", got)
+ }
+ a, b := uuid.New(), uuid.New()
+ if got := pgUUIDArray([]uuid.UUID{a, b}); got != "{"+a.String()+","+b.String()+"}" {
+ t.Errorf("two elements = %q", got)
+ }
+}
+
+func TestPropagation_TruncateError_RespectsCap(t *testing.T) {
+ if got := truncatePropagationError("short"); got != "short" {
+ t.Errorf("short changed: %q", got)
+ }
+ long := strings.Repeat("y", propagationLastErrorMax+50)
+ got := truncatePropagationError(long)
+ if len(got) != propagationLastErrorMax {
+ t.Errorf("truncated len = %d, want %d", len(got), propagationLastErrorMax)
+ }
+ if !strings.HasSuffix(got, "...") {
+ t.Errorf("expected ... suffix, got %q", got[len(got)-5:])
+ }
+}
+
+func TestPropagation_NullableTierString_EmptyAndValid(t *testing.T) {
+ if got := nullableTierString(sql.NullString{}); got != "" {
+ t.Errorf("invalid → %q", got)
+ }
+ if got := nullableTierString(sql.NullString{Valid: true, String: "pro"}); got != "pro" {
+ t.Errorf("valid → %q", got)
+ }
+}
+
+func TestPropagation_ResourceTypeFromString_AllArms(t *testing.T) {
+ cases := map[string]struct {
+ want commonv1.ResourceType
+ supported bool
+ }{
+ "postgres": {commonv1.ResourceType_RESOURCE_TYPE_POSTGRES, true},
+ "redis": {commonv1.ResourceType_RESOURCE_TYPE_REDIS, true},
+ "mongodb": {commonv1.ResourceType_RESOURCE_TYPE_MONGODB, true},
+ "storage": {commonv1.ResourceType_RESOURCE_TYPE_UNSPECIFIED, false},
+ "queue": {commonv1.ResourceType_RESOURCE_TYPE_UNSPECIFIED, false},
+ "webhook": {commonv1.ResourceType_RESOURCE_TYPE_UNSPECIFIED, false},
+ "future": {commonv1.ResourceType_RESOURCE_TYPE_UNSPECIFIED, false},
+ }
+ for in, want := range cases {
+ got, sup := resourceTypeFromString(in)
+ if got != want.want || sup != want.supported {
+ t.Errorf("resourceTypeFromString(%q) = (%v,%v), want (%v,%v)", in, got, sup, want.want, want.supported)
+ }
+ }
+}
+
+func TestPropagation_HandleTierElevation_EmptyTeamIsSuccess(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ mock.ExpectQuery(`FROM resources r`).
+ WithArgs(sqlmock.AnyArg()).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "token", "provider_resource_id", "tier", "resource_type",
+ }))
+
+ row := propagationRow{id: uuid.New(), teamID: uuid.New(), kind: "tier_elevation"}
+ regr := &stubPropagationRegrader{}
+ if err := handleTierElevation(context.Background(), db, regr, nil, row); err != nil {
+ t.Fatalf("empty team success: %v", err)
+ }
+ if regr.calls != 0 {
+ t.Errorf("expected 0 RegradeResource calls, got %d", regr.calls)
+ }
+}
+
+func TestPropagation_HandleTierElevation_QueryError_Returned(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ mock.ExpectQuery(`FROM resources r`).
+ WithArgs(sqlmock.AnyArg()).
+ WillReturnError(errors.New("select boom"))
+
+ row := propagationRow{id: uuid.New(), teamID: uuid.New(), kind: "tier_elevation"}
+ if err := handleTierElevation(context.Background(), db, &stubPropagationRegrader{}, nil, row); err == nil {
+ t.Fatal("expected error")
+ }
+}
+
+func TestPropagation_HandleTierElevation_UnsupportedAndEphemeral(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ // One storage row (unsupported) + one anonymous row (ephemeral) → both
+ // skipped → no regrader calls, no error.
+ mock.ExpectQuery(`FROM resources r`).
+ WithArgs(sqlmock.AnyArg()).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "token", "provider_resource_id", "tier", "resource_type",
+ }).
+ AddRow(uuid.New(), "tok1", "prid1", "pro", "storage").
+ AddRow(uuid.New(), "tok2", "prid2", "anonymous", "postgres"))
+
+ row := propagationRow{id: uuid.New(), teamID: uuid.New(), kind: "tier_elevation"}
+ regr := &stubPropagationRegrader{}
+ if err := handleTierElevation(context.Background(), db, regr, nil, row); err != nil {
+ t.Fatalf("unexpected err: %v", err)
+ }
+ if regr.calls != 0 {
+ t.Errorf("expected 0 regrade calls (storage skipped, ephemeral skipped), got %d", regr.calls)
+ }
+}
+
+func TestPropagation_HandleTierElevation_RegradeErrorReturnedFirst(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ mock.ExpectQuery(`FROM resources r`).
+ WithArgs(sqlmock.AnyArg()).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "token", "provider_resource_id", "tier", "resource_type",
+ }).AddRow(uuid.New(), "tok", "prid", "pro", "postgres"))
+
+ row := propagationRow{id: uuid.New(), teamID: uuid.New(), kind: "tier_elevation"}
+ regr := &stubPropagationRegrader{err: errors.New("grpc boom")}
+ err = handleTierElevation(context.Background(), db, regr, nil, row)
+ if err == nil {
+ t.Fatal("expected error")
+ }
+ if !strings.Contains(err.Error(), "grpc boom") {
+ t.Errorf("err text = %q", err.Error())
+ }
+}
+
+func TestPropagation_HandleTierElevation_AllowedSkipReason_NoError(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ mock.ExpectQuery(`FROM resources r`).
+ WithArgs(sqlmock.AnyArg()).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "token", "provider_resource_id", "tier", "resource_type",
+ }).AddRow(uuid.New(), "tok", "prid", "pro", "redis"))
+
+ row := propagationRow{id: uuid.New(), teamID: uuid.New(), kind: "tier_elevation"}
+ regr := &stubPropagationRegrader{outcome: regradeOutcome{Applied: false, SkipReason: "already correct"}}
+ if err := handleTierElevation(context.Background(), db, regr, nil, row); err != nil {
+ t.Fatalf("allowed skip must succeed: %v", err)
+ }
+}
+
+func TestPropagation_HandleTierElevation_ScanError_RowSkipped(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ // First row has bad UUID for id (will fail Scan), second row OK.
+ mock.ExpectQuery(`FROM resources r`).
+ WithArgs(sqlmock.AnyArg()).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "token", "provider_resource_id", "tier", "resource_type",
+ }).
+ AddRow("not-a-uuid", "tok", "prid", "pro", "postgres").
+ AddRow(uuid.New(), "tok", "prid", "pro", "postgres"))
+
+ row := propagationRow{id: uuid.New(), teamID: uuid.New(), kind: "tier_elevation"}
+ regr := &stubPropagationRegrader{outcome: regradeOutcome{Applied: true}}
+ if err := handleTierElevation(context.Background(), db, regr, nil, row); err != nil {
+ t.Fatalf("scan error: %v", err)
+ }
+ // Only the well-scanned row should be regraded.
+ if regr.calls != 1 {
+ t.Errorf("expected 1 regrade call, got %d", regr.calls)
+ }
+}
+
+func TestPropagation_BucketSkipReason_ExtraCases(t *testing.T) {
+ // Extra inputs for the bucketSkipReason switch arms not already tested by
+ // existing TestBucketSkipReason_BoundsCardinality.
+ cases := []struct {
+ in string
+ want string
+ }{
+ {"POSTGRES_ADMIN missing", "postgres_admin_secret_missing"},
+ {"some namespace not found here", "namespace_not_found"},
+ {"backend is not reachable", "resource_not_reachable"},
+ {"legacy pod has no creds", "legacy_resource"},
+ }
+ for _, c := range cases {
+ if got := bucketSkipReason(c.in); got != c.want {
+ t.Errorf("bucketSkipReason(%q) = %q, want %q", c.in, got, c.want)
+ }
+ }
+}
+
+// ─── geodb.go: Kind, constructor, Work + helpers ───────────────────────────────
+
+func TestGeoDB_Kind_ReturnsConstant(t *testing.T) {
+ if k := (RefreshGeoDBArgs{}).Kind(); k != "refresh_geodb" {
+ t.Errorf("Kind = %q", k)
+ }
+}
+
+func TestGeoDB_NewWorker_NonNil(t *testing.T) {
+ if w := NewRefreshGeoDBWorker(); w == nil {
+ t.Fatal("ctor returned nil")
+ }
+}
+
+func TestGeoDB_Work_NoLicenseKey_NoOp(t *testing.T) {
+ w := NewRefreshGeoDBWorker()
+ job := &river.Job[RefreshGeoDBArgs]{
+ Args: RefreshGeoDBArgs{LicenseKey: "", DBPath: "/tmp/nonexistent"},
+ JobRow: &rivertype.JobRow{ID: 1},
+ }
+ if err := w.Work(context.Background(), job); err != nil {
+ t.Fatalf("Work without license must noop: %v", err)
+ }
+}
+
+func TestGeoDB_Work_FreshDB_Skipped(t *testing.T) {
+ dir := t.TempDir()
+ path := filepath.Join(dir, "GeoLite2-City.mmdb")
+ if err := os.WriteFile(path, []byte("mmdb"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(path+".fetched", nil, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ w := NewRefreshGeoDBWorker()
+ job := &river.Job[RefreshGeoDBArgs]{
+ Args: RefreshGeoDBArgs{LicenseKey: "x", DBPath: path},
+ JobRow: &rivertype.JobRow{ID: 1},
+ }
+ if err := w.Work(context.Background(), job); err != nil {
+ t.Fatalf("fresh DB skip: %v", err)
+ }
+}
+
+func TestGeoDB_Work_BadDownloadURL_ReturnsError(t *testing.T) {
+ // LicenseKey set, DB not present, http GET will hit MaxMind which from
+ // the test environment we can't reach. The downloadURL has no scheme
+ // override; using an empty license generates URL fetch failure. Cancel
+ // the ctx fast so we don't actually dial maxmind.
+ w := NewRefreshGeoDBWorker()
+ job := &river.Job[RefreshGeoDBArgs]{
+ Args: RefreshGeoDBArgs{LicenseKey: "fake-license-key", DBPath: filepath.Join(t.TempDir(), "out.mmdb")},
+ JobRow: &rivertype.JobRow{ID: 1},
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
+ defer cancel()
+ // Either: network unreachable, or ctx deadline. Both surface as an error
+ // from Work, exercising the download-failed branch.
+ _ = w.Work(ctx, job)
+}
+
+func TestGeoDB_TouchFetchMarker_Roundtrip(t *testing.T) {
+ dir := t.TempDir()
+ path := filepath.Join(dir, "GeoLite2-City.mmdb")
+ touchGeoDBFetchMarker(path)
+ if _, err := os.Stat(path + geoLite2FetchMarkerSuffix); err != nil {
+ t.Fatalf("marker not written: %v", err)
+ }
+}
+
+func TestGeoDB_TouchFetchMarker_PermissionError_Logged(t *testing.T) {
+ // path under a non-existent directory → os.Create fails → branch
+ // logging warning but not panicking.
+ touchGeoDBFetchMarker("/nonexistent/dir/that/should/never/exist/file")
+}
+
+func TestGeoDB_IsFresh_EmptyPathReturnsFalse(t *testing.T) {
+ if geoDBIsFresh("", time.Now()) {
+ t.Error("empty path must be not-fresh")
+ }
+}
+
+// ─── team_deletion_executor.go ────────────────────────────────────────────────
+
+func TestTeamDeletion_Kind_ReturnsConstant(t *testing.T) {
+ if k := (TeamDeletionExecutorArgs{}).Kind(); k != "team_deletion_executor" {
+ t.Errorf("Kind = %q", k)
+ }
+}
+
+func TestTeamDeletion_BackupPrefix_EmptyTokenReturnsEmpty(t *testing.T) {
+ if got := s3BackupPrefixForToken(""); got != "" {
+ t.Errorf("empty token → %q, want \"\"", got)
+ }
+ tok := uuid.New().String()
+ if got := s3BackupPrefixForToken(tok); got != "backups/"+tok+"/" {
+ t.Errorf("normal token → %q", got)
+ }
+}
+
+func TestTeamDeletion_DeployNamespace_EmptyAppIDReturnsEmpty(t *testing.T) {
+ if got := deployNamespaceForApp(""); got != "" {
+ t.Errorf("empty appID → %q", got)
+ }
+ if got := deployNamespaceForApp("abc"); got != "instant-deploy-abc" {
+ t.Errorf("normal appID → %q", got)
+ }
+}
+
+func TestTeamDeletion_ContainsAny_AllArms(t *testing.T) {
+ if containsAny("hello world", "world") != true {
+ t.Error("substring should match")
+ }
+ if containsAny("hi", "longer-needle") != false {
+ t.Error("needle longer than haystack must be false")
+ }
+ if containsAny("xyz", "y") != true {
+ t.Error("single char match")
+ }
+ if containsAny("abc", "z") != false {
+ t.Error("missing match")
+ }
+}
+
+func TestTeamDeletion_NewExecutor_DefaultsBucket(t *testing.T) {
+ db, _, err := sqlmock.New()
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ w := NewTeamDeletionExecutorWorker(db, nil, nil, nil, "")
+ if w.bucketName != "instant-shared" {
+ t.Errorf("default bucket = %q, want instant-shared", w.bucketName)
+ }
+ w2 := NewTeamDeletionExecutorWorker(db, nil, nil, nil, "custom-bucket")
+ if w2.bucketName != "custom-bucket" {
+ t.Errorf("custom bucket = %q", w2.bucketName)
+ }
+}
+
+func TestTeamDeletion_FetchCandidates_QueryError(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ mock.ExpectQuery(`FROM teams\s+WHERE`).WillReturnError(errors.New("query failed"))
+
+ w := NewTeamDeletionExecutorWorker(db, nil, nil, nil, "")
+ if err := w.Work(context.Background(), localJob[TeamDeletionExecutorArgs]()); err == nil {
+ t.Fatal("expected error from top-level query failure")
+ }
+}
+
+func TestTeamDeletion_EmitDeletionFailed_StepInference_AllArms(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ w := NewTeamDeletionExecutorWorker(db, nil, nil, nil, "")
+ teamID := uuid.New()
+
+ cases := []struct {
+ err error
+ step string
+ }{
+ {context.Canceled, "context"},
+ {context.DeadlineExceeded, "context"},
+ {errors.New("fetch resources: db error"), "fetch_resources"},
+ {errors.New("delete s3 backups for x"), "s3_delete"},
+ {errors.New("deprovision postgres: failed"), "deprovision"},
+ {errors.New("delete namespace foo: bad"), "delete_namespace"},
+ {errors.New("fetch deploy app ids: query"), "delete_namespace"},
+ {errors.New("mark deletion_pending: x"), "mark_deletion_pending"},
+ {errors.New("null resource pii: oops"), "null_resource_pii"},
+ {errors.New("null user pii: oops"), "null_user_pii"},
+ {errors.New("flip team status: oops"), "flip_team_status"},
+ {errors.New("commit tx: oops"), "tx_commit"},
+ {errors.New("begin tx: oops"), "tx_commit"},
+ {errors.New("totally unrelated error"), "unknown"},
+ }
+ for _, c := range cases {
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs(teamID, "system", auditKindTeamDeletionFailed, sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ w.emitDeletionFailed(context.Background(), teamID, c.err)
+ }
+ if err := mock.ExpectationsWereMet(); err != nil {
+ t.Errorf("unmet: %v", err)
+ }
+}
+
+func TestTeamDeletion_EmitDeletionFailed_InsertError_Logged(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ w := NewTeamDeletionExecutorWorker(db, nil, nil, nil, "")
+ teamID := uuid.New()
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs(teamID, "system", auditKindTeamDeletionFailed, sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnError(errors.New("audit insert failed"))
+ w.emitDeletionFailed(context.Background(), teamID, errors.New("upstream"))
+}
+
+func TestTeamDeletion_EmitTombstoned_InsertError_Logged(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ w := NewTeamDeletionExecutorWorker(db, nil, nil, nil, "")
+ teamID := uuid.New()
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs(teamID, "system", auditKindTombstoned, sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnError(errors.New("insert failed"))
+ w.emitTombstoned(context.Background(), teamID, 1, 0, 0, time.Second)
+}
+
+func TestTeamDeletion_FetchTeamDeployAppIDs_QueryError(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ w := NewTeamDeletionExecutorWorker(db, nil, nil, nil, "")
+ mock.ExpectQuery(`SELECT DISTINCT app_id`).
+ WithArgs(sqlmock.AnyArg()).
+ WillReturnError(errors.New("boom"))
+ if _, err := w.fetchTeamDeployAppIDs(context.Background(), uuid.New()); err == nil {
+ t.Fatal("expected error")
+ }
+}
+
+func TestTeamDeletion_FetchTeamResources_QueryError(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ w := NewTeamDeletionExecutorWorker(db, nil, nil, nil, "")
+ mock.ExpectQuery(`FROM resources`).
+ WithArgs(sqlmock.AnyArg()).
+ WillReturnError(errors.New("boom"))
+ if _, err := w.fetchTeamResources(context.Background(), uuid.New()); err == nil {
+ t.Fatal("expected error")
+ }
+}
+
+func TestTeamDeletion_FetchCandidates_ScanError(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ w := NewTeamDeletionExecutorWorker(db, nil, nil, nil, "")
+ // Bad UUID + good time → scan fails.
+ mock.ExpectQuery(`FROM teams\s+WHERE`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "deletion_requested_at"}).
+ AddRow("not-a-uuid", time.Now()))
+ if _, err := w.fetchCandidates(context.Background()); err == nil {
+ t.Fatal("expected scan error")
+ }
+}
+
+// ─── team_deletion_audit_kinds.go ─────────────────────────────────────────────
+
+func TestAuditKind_TombstonedAndFailed_Constants(t *testing.T) {
+ if auditKindTombstoned != "team.tombstoned" {
+ t.Errorf("auditKindTombstoned = %q", auditKindTombstoned)
+ }
+ if auditKindTeamDeletionFailed != "team.deletion_failed" {
+ t.Errorf("auditKindTeamDeletionFailed = %q", auditKindTeamDeletionFailed)
+ }
+}
+
+func TestAuditKind_ChaosLeaseRecovery_Constants(t *testing.T) {
+ if AuditKindChaosLeaseRecoveryStart != "chaos.lease_recovery.start" {
+ t.Errorf("start = %q", AuditKindChaosLeaseRecoveryStart)
+ }
+ if AuditKindChaosLeaseRecoveryEnd != "chaos.lease_recovery.end" {
+ t.Errorf("end = %q", AuditKindChaosLeaseRecoveryEnd)
+ }
+}
+
+// ─── team_deletion_s3_adapter.go ──────────────────────────────────────────────
+
+func TestS3Adapter_NewMinIOBackupDeleter_NilScannerReturnsNil(t *testing.T) {
+ if got := newMinIOBackupDeleter(nil); got != nil {
+ t.Errorf("nil scanner → %v, want nil", got)
+ }
+}
+
+// localFakeObjectLister satisfies minioObjectLister with no-ops, lets us
+// drive newMinIOScannerWithClient via the test seam.
+type localFakeObjectLister struct{}
+
+func (localFakeObjectLister) BucketExists(_ context.Context, _ string) (bool, error) {
+ return true, nil
+}
+func (localFakeObjectLister) ListObjects(_ context.Context, _ string, _ minio.ListObjectsOptions) <-chan minio.ObjectInfo {
+ ch := make(chan minio.ObjectInfo)
+ close(ch)
+ return ch
+}
+func (localFakeObjectLister) ListIncompleteUploads(_ context.Context, _, _ string, _ bool) <-chan minio.ObjectMultipartInfo {
+ ch := make(chan minio.ObjectMultipartInfo)
+ close(ch)
+ return ch
+}
+
+func TestS3Adapter_NewMinIOBackupDeleter_FakeClientReturnsNil(t *testing.T) {
+ // Scanner with a fake (non-*minio.Client) → the type assertion fails →
+ // the adapter returns nil per its contract (test seam path).
+ scanner := newMinIOScannerWithClient(localFakeObjectLister{}, "test-bucket")
+ if got := newMinIOBackupDeleter(scanner); got != nil {
+ t.Errorf("fake client → %v, want nil", got)
+ }
+}
+
+func TestS3Adapter_DeleterListAndRemove_RoundTrip(t *testing.T) {
+ // Real *minio.Client constructed against an unreachable endpoint —
+ // we only exercise the wrapper methods, NOT the underlying RPCs.
+ mc, err := minio.New("127.0.0.1:1", &minio.Options{Secure: false})
+ if err != nil {
+ t.Fatal(err)
+ }
+ d := &minioBackupDeleter{client: mc, bucketName: "any"}
+ ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
+ defer cancel()
+ // ListObjects returns a channel — drain it to exercise the call path.
+ ch := d.ListObjects(ctx, "any", minio.ListObjectsOptions{})
+ for range ch {
+ }
+ // RemoveObjects accepts an input channel — close it immediately.
+ in := make(chan minio.ObjectInfo)
+ close(in)
+ errCh := d.RemoveObjects(ctx, "any", in, minio.RemoveObjectsOptions{})
+ for range errCh {
+ }
+}
+
+func TestS3Adapter_NewMinIOBackupDeleter_RealClient_NotNil(t *testing.T) {
+ // Build a *minio.Client by hand and wrap it in a scanner. The factory
+ // should now succeed (return non-nil).
+ mc, err := minio.New("127.0.0.1:1", &minio.Options{Secure: false})
+ if err != nil {
+ t.Fatal(err)
+ }
+ scanner := newMinIOScannerWithClient(mc, "test")
+ if got := newMinIOBackupDeleter(scanner); got == nil {
+ t.Error("real minio.Client → adapter should be non-nil")
+ }
+}
+
+// ─── chaos_lease_recovery.go ──────────────────────────────────────────────────
+
+func TestChaos_LeaseRecovery_Kind(t *testing.T) {
+ if k := (ChaosLeaseRecoveryArgs{}).Kind(); k != chaosLeaseRecoveryKind {
+ t.Errorf("Kind = %q", k)
+ }
+}
+
+func TestChaos_LeaseRecovery_InsertOpts(t *testing.T) {
+ opts := (ChaosLeaseRecoveryArgs{}).InsertOpts()
+ if opts.Queue != river.QueueDefault {
+ t.Errorf("queue = %q", opts.Queue)
+ }
+ if opts.Priority != 4 {
+ t.Errorf("priority = %d, want 4", opts.Priority)
+ }
+}
+
+func TestChaos_LeaseRecovery_NewWorker(t *testing.T) {
+ db, _, _ := sqlmock.New()
+ defer db.Close()
+ if w := NewChaosLeaseRecoveryWorker(db); w == nil {
+ t.Fatal("ctor returned nil")
+ }
+}
+
+func TestChaos_LeaseRecovery_PodHostname(t *testing.T) {
+ prev := os.Getenv("HOSTNAME")
+ defer os.Setenv("HOSTNAME", prev)
+
+ os.Setenv("HOSTNAME", "worker-pod-42")
+ if got := podHostname(); got != "worker-pod-42" {
+ t.Errorf("set → %q", got)
+ }
+ os.Unsetenv("HOSTNAME")
+ if got := podHostname(); got != "unknown" {
+ t.Errorf("unset → %q", got)
+ }
+}
+
+func TestChaos_LeaseRecovery_Work_BadTeamID_Errors(t *testing.T) {
+ db, _, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ w := NewChaosLeaseRecoveryWorker(db)
+ job := &river.Job[ChaosLeaseRecoveryArgs]{
+ Args: ChaosLeaseRecoveryArgs{TeamID: "not-a-uuid", SleepSeconds: 0, RunID: "run1"},
+ JobRow: &rivertype.JobRow{ID: 1},
+ }
+ if err := w.Work(context.Background(), job); err == nil {
+ t.Fatal("expected error for bad team UUID")
+ }
+}
+
+func TestChaos_LeaseRecovery_Work_HappyPath_Sleep0(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ teamID := uuid.New().String()
+
+ // Two INSERT INTO audit_log calls — start + end.
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs(sqlmock.AnyArg(), chaosLeaseRecoveryActor,
+ AuditKindChaosLeaseRecoveryStart, "drill start", sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs(sqlmock.AnyArg(), chaosLeaseRecoveryActor,
+ AuditKindChaosLeaseRecoveryEnd, "drill end", sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(2, 1))
+
+ w := NewChaosLeaseRecoveryWorker(db)
+ job := &river.Job[ChaosLeaseRecoveryArgs]{
+ Args: ChaosLeaseRecoveryArgs{TeamID: teamID, SleepSeconds: 0, RunID: "run-happy"},
+ JobRow: &rivertype.JobRow{ID: 1},
+ }
+ if err := w.Work(context.Background(), job); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+ if err := mock.ExpectationsWereMet(); err != nil {
+ t.Errorf("unmet: %v", err)
+ }
+}
+
+func TestChaos_LeaseRecovery_Work_NegativeSleepClampedToZero(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ teamID := uuid.New().String()
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs(sqlmock.AnyArg(), chaosLeaseRecoveryActor,
+ AuditKindChaosLeaseRecoveryStart, sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs(sqlmock.AnyArg(), chaosLeaseRecoveryActor,
+ AuditKindChaosLeaseRecoveryEnd, sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(2, 1))
+ w := NewChaosLeaseRecoveryWorker(db)
+ job := &river.Job[ChaosLeaseRecoveryArgs]{
+ Args: ChaosLeaseRecoveryArgs{TeamID: teamID, SleepSeconds: -5},
+ JobRow: &rivertype.JobRow{ID: 1},
+ }
+ if err := w.Work(context.Background(), job); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+}
+
+func TestChaos_LeaseRecovery_Work_StartMarkerError_Returns(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ teamID := uuid.New().String()
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs(sqlmock.AnyArg(), chaosLeaseRecoveryActor,
+ AuditKindChaosLeaseRecoveryStart, sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnError(errors.New("audit insert failed"))
+ w := NewChaosLeaseRecoveryWorker(db)
+ job := &river.Job[ChaosLeaseRecoveryArgs]{
+ Args: ChaosLeaseRecoveryArgs{TeamID: teamID, SleepSeconds: 0},
+ JobRow: &rivertype.JobRow{ID: 1},
+ }
+ if err := w.Work(context.Background(), job); err == nil {
+ t.Fatal("expected error when start marker fails")
+ }
+}
+
+func TestChaos_LeaseRecovery_Work_EndMarkerError_Returns(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ teamID := uuid.New().String()
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs(sqlmock.AnyArg(), chaosLeaseRecoveryActor,
+ AuditKindChaosLeaseRecoveryStart, sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs(sqlmock.AnyArg(), chaosLeaseRecoveryActor,
+ AuditKindChaosLeaseRecoveryEnd, sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnError(errors.New("audit insert failed"))
+ w := NewChaosLeaseRecoveryWorker(db)
+ job := &river.Job[ChaosLeaseRecoveryArgs]{
+ Args: ChaosLeaseRecoveryArgs{TeamID: teamID, SleepSeconds: 0},
+ JobRow: &rivertype.JobRow{ID: 1},
+ }
+ if err := w.Work(context.Background(), job); err == nil {
+ t.Fatal("expected error when end marker fails")
+ }
+}
+
+func TestChaos_LeaseRecovery_Work_CtxCancelledDuringSleep(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ teamID := uuid.New().String()
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs(sqlmock.AnyArg(), chaosLeaseRecoveryActor,
+ AuditKindChaosLeaseRecoveryStart, sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ w := NewChaosLeaseRecoveryWorker(db)
+ job := &river.Job[ChaosLeaseRecoveryArgs]{
+ Args: ChaosLeaseRecoveryArgs{TeamID: teamID, SleepSeconds: 60},
+ JobRow: &rivertype.JobRow{ID: 1},
+ }
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel() // already cancelled — ctx.Done() fires immediately
+ if err := w.Work(ctx, job); err == nil {
+ t.Fatal("expected ctx error")
+ }
+}
+
+func TestChaos_LeaseRecovery_Work_SleepClampedToMax(t *testing.T) {
+ // SleepSeconds way over the cap — exercises the cap branch. We don't
+ // actually wait the cap (5min); we cancel ctx fast.
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ teamID := uuid.New().String()
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs(sqlmock.AnyArg(), chaosLeaseRecoveryActor,
+ AuditKindChaosLeaseRecoveryStart, sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ w := NewChaosLeaseRecoveryWorker(db)
+ job := &river.Job[ChaosLeaseRecoveryArgs]{
+ Args: ChaosLeaseRecoveryArgs{TeamID: teamID, SleepSeconds: 100000},
+ JobRow: &rivertype.JobRow{ID: 1},
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
+ defer cancel()
+ _ = w.Work(ctx, job)
+}
+
+// ─── uptime_prober.go: Kind, dialer ──────────────────────────────────────────
+
+func TestProber_Uptime_Kind(t *testing.T) {
+ if k := (UptimeProberArgs{}).Kind(); k != "uptime_prober" {
+ t.Errorf("Kind = %q", k)
+ }
+}
+
+func TestProber_UptimeRetention_Kind(t *testing.T) {
+ if k := (UptimeRetentionArgs{}).Kind(); k != "uptime_retention" {
+ t.Errorf("Kind = %q", k)
+ }
+}
+
+func TestProber_Uptime_DefaultDialer_UnreachableErrors(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
+ defer cancel()
+ err := defaultProvisionerDialer(ctx, "127.0.0.1:1")
+ if err == nil {
+ t.Fatal("expected dial error to an unreachable port")
+ }
+}
+
+func TestProber_UptimeRetention_DBError_Propagates(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ mock.ExpectExec(`DELETE FROM uptime_samples`).WillReturnError(errors.New("boom"))
+ w := NewUptimeRetentionWorker(db)
+ if err := w.Work(context.Background(), localJob[UptimeRetentionArgs]()); err == nil {
+ t.Fatal("expected error from delete failure")
+ }
+}
+
+// ─── real_prober.go ────────────────────────────────────────────────────────────
+
+func TestProber_NormalizeStorageURL_AllArms(t *testing.T) {
+ if _, err := url.Parse("://bad"); err == nil {
+ // just here to silence the unused-import linter if it ever fires
+ }
+
+ // http/https → returned untouched.
+ if got, err := normalizeStorageURL("https://example.com/bucket"); err != nil || got != "https://example.com/bucket" {
+ t.Errorf("https returned (%q, %v)", got, err)
+ }
+ if got, err := normalizeStorageURL("http://example.com/bucket"); err != nil || got != "http://example.com/bucket" {
+ t.Errorf("http returned (%q, %v)", got, err)
+ }
+
+ // s3:///prefix → https://.s3.amazonaws.com/
+ got, err := normalizeStorageURL("s3://mybucket/prefix")
+ if err != nil || got != "https://mybucket.s3.amazonaws.com/" {
+ t.Errorf("s3 returned (%q, %v)", got, err)
+ }
+
+ // s3:// without host → error.
+ if _, err := normalizeStorageURL("s3:///path"); err == nil {
+ t.Error("missing-bucket s3 must error")
+ }
+
+ // Unknown scheme → error.
+ if _, err := normalizeStorageURL("ftp://example.com/"); err == nil {
+ t.Error("unsupported scheme must error")
+ }
+
+ // Bad URL → error.
+ if _, err := normalizeStorageURL("://nope"); err == nil {
+ t.Error("malformed URL must error")
+ }
+}
+
+func TestProber_NatsHost_AllArms(t *testing.T) {
+ if h, err := natsHost("nats://server.example.com:4222"); err != nil || h != "server.example.com" {
+ t.Errorf("nats:// returned (%q, %v)", h, err)
+ }
+ if h, err := natsHost("tls://server.example.com:4222"); err != nil || h != "server.example.com" {
+ t.Errorf("tls:// returned (%q, %v)", h, err)
+ }
+ if _, err := natsHost("http://x"); err == nil {
+ t.Error("non-nats scheme must error")
+ }
+ if _, err := natsHost("nats://"); err == nil {
+ t.Error("missing host must error")
+ }
+ if _, err := natsHost("://"); err == nil {
+ t.Error("malformed URL must error")
+ }
+}
+
+func TestProber_LooksLikePlaintextURL_AllSchemes(t *testing.T) {
+ yes := []string{
+ "postgres://x", "postgresql://x", "redis://x", "rediss://x",
+ "mongodb://x", "mongodb+srv://x", "http://x", "https://x",
+ "nats://x", "s3://x",
+ }
+ for _, s := range yes {
+ if !looksLikePlaintextURL(s) {
+ t.Errorf("looksLikePlaintextURL(%q) = false, want true", s)
+ }
+ }
+ for _, s := range []string{"ftp://x", "weird", "://x", " postgres://x"} {
+ if looksLikePlaintextURL(s) {
+ t.Errorf("looksLikePlaintextURL(%q) = true, want false", s)
+ }
+ }
+}
+
+func TestProber_NetError_Reporting(t *testing.T) {
+ // netError is marked //nolint:unused but reachable via the export — we
+ // can call it directly here in-package.
+ if netError(nil) {
+ t.Error("nil → false")
+ }
+ if netError(errors.New("plain")) {
+ t.Error("plain error → false")
+ }
+ if !netError(&net_error_for_test{msg: "x"}) {
+ t.Error("net.Error → true")
+ }
+}
+
+// net_error_for_test implements net.Error so netError can return true.
+type net_error_for_test struct{ msg string }
+
+func (e *net_error_for_test) Error() string { return e.msg }
+func (e *net_error_for_test) Timeout() bool { return false }
+func (e *net_error_for_test) Temporary() bool { return false }
+
+func TestProber_RealProber_DecryptEmptyURL(t *testing.T) {
+ p := NewRealProber(&config.Config{AESKey: ""}).(*realProber)
+ if _, err := p.decrypt(""); err == nil {
+ t.Error("empty url → error expected")
+ }
+}
+
+func TestProber_RealProber_DecryptNilKeyReturnsAsIs(t *testing.T) {
+ p := NewRealProber(&config.Config{AESKey: ""}).(*realProber)
+ got, err := p.decrypt("postgres://anything")
+ if err != nil || got != "postgres://anything" {
+ t.Errorf("nil-key path: (%q, %v)", got, err)
+ }
+}
+
+func TestProber_RealProber_ProbeStorage_BadURL(t *testing.T) {
+ p := NewRealProber(&config.Config{AESKey: ""}).(*realProber)
+ out, err := p.probeStorage(context.Background(), "ftp://nope/")
+ if out != ProbeUnreachable || err == nil {
+ t.Errorf("bad URL: (%v, %v)", out, err)
+ }
+}
+
+func TestProber_RealProber_ProbeQueue_BadURL(t *testing.T) {
+ p := NewRealProber(&config.Config{AESKey: ""}).(*realProber)
+ out, err := p.probeQueue(context.Background(), "ftp://nope/")
+ if out != ProbeUnreachable || err == nil {
+ t.Errorf("bad URL: (%v, %v)", out, err)
+ }
+}
+
+func TestProber_RealProber_ProbeQueue_NonHealthyHTTP(t *testing.T) {
+ p := NewRealProber(&config.Config{AESKey: ""}).(*realProber)
+ // nats://192.0.2.1 → builds http://192.0.2.1:8222/healthz → unreachable
+ // blackhole → ProbeUnreachable via the GET error branch.
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+ defer cancel()
+ out, _ := p.probeQueue(ctx, "nats://192.0.2.1:4222")
+ if out != ProbeUnreachable {
+ t.Errorf("blackhole nats: %v", out)
+ }
+}
+
+func TestProber_RealProber_ProbePostgres_BadConn(t *testing.T) {
+ p := NewRealProber(&config.Config{AESKey: ""}).(*realProber)
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+ defer cancel()
+ // Malformed postgres URL → sql.Open succeeds (lazy), QueryRowContext
+ // fails fast → ProbeUnreachable.
+ out, err := p.probePostgres(ctx, "postgres://u:p@127.0.0.1:1/db?sslmode=disable&connect_timeout=1")
+ if out != ProbeUnreachable {
+ t.Errorf("unreachable pg: (%v, %v)", out, err)
+ }
+}
+
+func TestProber_RealProber_ProbeRedis_BadURL(t *testing.T) {
+ p := NewRealProber(&config.Config{AESKey: ""}).(*realProber)
+ out, err := p.probeRedis(context.Background(), "not a url")
+ if out != ProbeUnreachable {
+ t.Errorf("bad url: (%v, %v)", out, err)
+ }
+}
+
+func TestProber_RealProber_ProbeMongo_BadURL(t *testing.T) {
+ p := NewRealProber(&config.Config{AESKey: ""}).(*realProber)
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+ defer cancel()
+ out, _ := p.probeMongo(ctx, "mongodb://u:p@127.0.0.1:1/?serverSelectionTimeoutMS=200")
+ if out != ProbeUnreachable {
+ t.Errorf("unreachable mongo: %v", out)
+ }
+}
+
+// ─── uptime_prober.go: full Work() + insertSample + each probe (matching the
+// brief's TestProber filter so they actually run under it) ─────────────
+
+func TestProber_Uptime_NewWorker_NonNil(t *testing.T) {
+ db, _, _ := sqlmock.New()
+ defer db.Close()
+ if w := NewUptimeProberWorker(db); w == nil {
+ t.Fatal("ctor returned nil")
+ }
+}
+
+func TestProber_Uptime_SetDialer(t *testing.T) {
+ db, _, _ := sqlmock.New()
+ defer db.Close()
+ w := NewUptimeProberWorker(db)
+ called := false
+ SetUptimeProberDialer(w, func(_ context.Context, _ string) error {
+ called = true
+ return nil
+ })
+ _ = w.provisionerDialer(context.Background(), "x")
+ if !called {
+ t.Error("custom dialer not called")
+ }
+}
+
+func TestProber_EnvOr_Behavior(t *testing.T) {
+ t.Setenv("UPTIME_TEST_VAR", " hello ")
+ if got := envOr("UPTIME_TEST_VAR", "fb"); got != "hello" {
+ t.Errorf("trim → %q", got)
+ }
+ t.Setenv("UPTIME_TEST_VAR", "")
+ if got := envOr("UPTIME_TEST_VAR", "fb"); got != "fb" {
+ t.Errorf("empty → %q", got)
+ }
+ t.Setenv("UPTIME_TEST_VAR", " ")
+ if got := envOr("UPTIME_TEST_VAR", "fb"); got != "fb" {
+ t.Errorf("whitespace → %q", got)
+ }
+}
+
+func TestProber_Uptime_InsertSample_DBError(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ mock.ExpectExec(`INSERT INTO uptime_samples`).
+ WillReturnError(errors.New("insert failed"))
+ w := NewUptimeProberWorker(db)
+ if err := w.insertSample(context.Background(), "api", true, nil); err == nil {
+ t.Fatal("expected error")
+ }
+}
+
+func TestProber_Uptime_InsertSample_WithLatency(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ mock.ExpectExec(`INSERT INTO uptime_samples`).
+ WithArgs("api", true, sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ w := NewUptimeProberWorker(db)
+ latency := 25
+ if err := w.insertSample(context.Background(), "api", true, &latency); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestProber_Uptime_Work_AllProbesSucceed(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ mock.MatchExpectationsInOrder(false)
+
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ }))
+ defer srv.Close()
+ t.Setenv("UPTIME_PROBE_API_URL", srv.URL+"/healthz")
+ t.Setenv("UPTIME_PROBE_MARKETING_URL", srv.URL+"/")
+ t.Setenv("UPTIME_PROBE_DEPLOYS_URL", srv.URL+"/")
+
+ mock.ExpectQuery(`SELECT 1`).
+ WillReturnRows(sqlmock.NewRows([]string{"?column?"}).AddRow(1))
+ for _, slug := range []string{"api", "provisioner", "worker", "deploys", "marketing"} {
+ mock.ExpectExec(`INSERT INTO uptime_samples`).
+ WithArgs(slug, true, sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ }
+
+ w := NewUptimeProberWorker(db)
+ SetUptimeProberDialer(w, func(_ context.Context, _ string) error { return nil })
+ if err := w.Work(context.Background(), localJob[UptimeProberArgs]()); err != nil {
+ t.Fatal(err)
+ }
+ if err := mock.ExpectationsWereMet(); err != nil {
+ t.Errorf("unmet: %v", err)
+ }
+}
+
+func TestProber_Uptime_Work_FailedInsertContinues(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ mock.MatchExpectationsInOrder(false)
+
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ }))
+ defer srv.Close()
+ t.Setenv("UPTIME_PROBE_API_URL", srv.URL+"/healthz")
+ t.Setenv("UPTIME_PROBE_MARKETING_URL", srv.URL+"/")
+ t.Setenv("UPTIME_PROBE_DEPLOYS_URL", srv.URL+"/")
+
+ mock.ExpectQuery(`SELECT 1`).
+ WillReturnRows(sqlmock.NewRows([]string{"?column?"}).AddRow(1))
+ // All inserts fail — Work must still return nil (log + continue).
+ for i := 0; i < 5; i++ {
+ mock.ExpectExec(`INSERT INTO uptime_samples`).
+ WillReturnError(errors.New("insert failed"))
+ }
+
+ w := NewUptimeProberWorker(db)
+ SetUptimeProberDialer(w, func(_ context.Context, _ string) error { return nil })
+ if err := w.Work(context.Background(), localJob[UptimeProberArgs]()); err != nil {
+ t.Fatalf("insert failures must not propagate: %v", err)
+ }
+}
+
+func TestProber_Uptime_ProbeWorker_SelectFails(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ mock.MatchExpectationsInOrder(false)
+
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ }))
+ defer srv.Close()
+ t.Setenv("UPTIME_PROBE_API_URL", srv.URL+"/healthz")
+ t.Setenv("UPTIME_PROBE_MARKETING_URL", srv.URL+"/")
+ t.Setenv("UPTIME_PROBE_DEPLOYS_URL", srv.URL+"/")
+
+ mock.ExpectQuery(`SELECT 1`).
+ WillReturnError(errors.New("db down"))
+ for _, slug := range []string{"api", "provisioner", "worker", "deploys", "marketing"} {
+ mock.ExpectExec(`INSERT INTO uptime_samples`).
+ WithArgs(slug, sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ }
+ w := NewUptimeProberWorker(db)
+ SetUptimeProberDialer(w, func(_ context.Context, _ string) error { return nil })
+ if err := w.Work(context.Background(), localJob[UptimeProberArgs]()); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestProber_Uptime_HttpHEAD_UnreachableServer(t *testing.T) {
+ db, _, _ := sqlmock.New()
+ defer db.Close()
+ w := NewUptimeProberWorker(db)
+ srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {}))
+ srv.Close() // closed → unreachable
+ healthy, latency := w.httpHEAD(context.Background(), srv.URL+"/", false)
+ if healthy {
+ t.Error("expected unhealthy")
+ }
+ if latency != nil {
+ t.Errorf("expected nil latency, got %v", latency)
+ }
+}
+
+func TestProber_Uptime_HttpHEAD_BadURL(t *testing.T) {
+ db, _, _ := sqlmock.New()
+ defer db.Close()
+ w := NewUptimeProberWorker(db)
+ // http.NewRequestWithContext rejects URLs with bad control characters.
+ healthy, latency := w.httpHEAD(context.Background(), "http://\x00bad", true)
+ if healthy {
+ t.Error("expected unhealthy on bad URL")
+ }
+ if latency != nil {
+ t.Errorf("expected nil latency, got %v", latency)
+ }
+}
+
+func TestProber_UptimeRetention_HappyPath(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ mock.ExpectExec(`DELETE FROM uptime_samples`).
+ WithArgs(90).
+ WillReturnResult(sqlmock.NewResult(0, 17))
+ w := NewUptimeRetentionWorker(db)
+ if err := w.Work(context.Background(), localJob[UptimeRetentionArgs]()); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// ─── More propagation_runner / heartbeat / geodb / executor coverage ─────────
+
+func TestPropagation_MarkApplied_DBError_Returns(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ mock.ExpectExec(`UPDATE pending_propagations\s+SET applied_at`).
+ WillReturnError(errors.New("update failed"))
+ w := NewPropagationRunnerWorker(db, nil, &stubPropagationRegrader{})
+ row := propagationRow{id: uuid.New(), teamID: uuid.New(), kind: "tier_elevation"}
+ if err := w.markApplied(context.Background(), row); err == nil {
+ t.Fatal("expected error")
+ }
+}
+
+func TestPropagation_MarkApplied_NoOpZeroRows(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ mock.ExpectExec(`UPDATE pending_propagations\s+SET applied_at`).
+ WillReturnResult(sqlmock.NewResult(0, 0)) // 0 rows → sibling raced
+ w := NewPropagationRunnerWorker(db, nil, &stubPropagationRegrader{})
+ row := propagationRow{id: uuid.New(), teamID: uuid.New(), kind: "tier_elevation"}
+ if err := w.markApplied(context.Background(), row); err != nil {
+ t.Fatalf("0 rows must be nil: %v", err)
+ }
+}
+
+func TestPropagation_MarkRetry_DBError_Logged(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ mock.ExpectExec(`UPDATE pending_propagations\s+SET attempts`).
+ WillReturnError(errors.New("retry update failed"))
+ w := NewPropagationRunnerWorker(db, nil, &stubPropagationRegrader{})
+ w.now = func() time.Time { return time.Now() }
+ row := propagationRow{id: uuid.New(), teamID: uuid.New(), kind: "tier_elevation"}
+ w.markRetry(context.Background(), row, errors.New("dispatch fail"))
+}
+
+func TestPropagation_MarkRetry_NoOpZeroRows(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ mock.ExpectExec(`UPDATE pending_propagations\s+SET attempts`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ w := NewPropagationRunnerWorker(db, nil, &stubPropagationRegrader{})
+ w.now = func() time.Time { return time.Now() }
+ row := propagationRow{id: uuid.New(), teamID: uuid.New(), kind: "tier_elevation"}
+ w.markRetry(context.Background(), row, errors.New("dispatch fail"))
+}
+
+func TestPropagation_MarkRetry_UnexpectedSkip_AuditUsesDistinctKind(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ mock.ExpectExec(`UPDATE pending_propagations\s+SET attempts`).
+ WillReturnResult(sqlmock.NewResult(0, 1))
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs(sqlmock.AnyArg(), propagationActor, "propagation.unexpected_skip", sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ w := NewPropagationRunnerWorker(db, nil, &stubPropagationRegrader{})
+ w.now = func() time.Time { return time.Now() }
+ row := propagationRow{id: uuid.New(), teamID: uuid.New(), kind: "tier_elevation"}
+ skipErr := &propagationUnexpectedSkipErr{
+ Resources: []propagationUnexpectedSkipDetail{
+ {ResourceID: "r1", ResourceType: "postgres", SkipReason: "postgres-admin secret not found"},
+ },
+ }
+ w.markRetry(context.Background(), row, skipErr)
+}
+
+func TestPropagation_MarkDeadLettered_DBError_Logged(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ mock.ExpectExec(`UPDATE pending_propagations\s+SET attempts`).
+ WillReturnError(errors.New("dead-letter update failed"))
+ w := NewPropagationRunnerWorker(db, nil, &stubPropagationRegrader{})
+ w.now = func() time.Time { return time.Now() }
+ row := propagationRow{id: uuid.New(), teamID: uuid.New(), kind: "tier_elevation"}
+ w.markDeadLettered(context.Background(), row, errors.New("terminal"))
+}
+
+func TestPropagation_MarkDeadLettered_NoOpZeroRows(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ mock.ExpectExec(`UPDATE pending_propagations\s+SET attempts`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ w := NewPropagationRunnerWorker(db, nil, &stubPropagationRegrader{})
+ w.now = func() time.Time { return time.Now() }
+ row := propagationRow{id: uuid.New(), teamID: uuid.New(), kind: "tier_elevation"}
+ w.markDeadLettered(context.Background(), row, errors.New("terminal"))
+}
+
+func TestPropagation_MarkUnknownKindDeadLettered_DBError(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ mock.ExpectExec(`UPDATE pending_propagations\s+SET attempts`).
+ WillReturnError(errors.New("unknown-kind update failed"))
+ w := NewPropagationRunnerWorker(db, nil, &stubPropagationRegrader{})
+ w.now = func() time.Time { return time.Now() }
+ row := propagationRow{id: uuid.New(), teamID: uuid.New(), kind: "future_kind"}
+ w.markUnknownKindDeadLettered(context.Background(), row, errors.New("no handler"))
+}
+
+func TestPropagation_MarkUnknownKindDeadLettered_NoOpZeroRows(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ mock.ExpectExec(`UPDATE pending_propagations\s+SET attempts`).
+ WillReturnResult(sqlmock.NewResult(0, 0))
+ w := NewPropagationRunnerWorker(db, nil, &stubPropagationRegrader{})
+ w.now = func() time.Time { return time.Now() }
+ row := propagationRow{id: uuid.New(), teamID: uuid.New(), kind: "future_kind"}
+ w.markUnknownKindDeadLettered(context.Background(), row, errors.New("no handler"))
+}
+
+func TestPropagation_InsertAuditRow_DBError_Logged(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WillReturnError(errors.New("audit insert failed"))
+ w := NewPropagationRunnerWorker(db, nil, &stubPropagationRegrader{})
+ w.insertPropagationAuditRow(context.Background(),
+ propagationRow{id: uuid.New(), teamID: uuid.New(), kind: "tier_elevation"},
+ "propagation.applied", "summary", map[string]any{"k": "v"})
+}
+
+func TestPropagation_UnexpectedSkipErr_EmptyError(t *testing.T) {
+ var e *propagationUnexpectedSkipErr
+ if got := e.Error(); !strings.Contains(got, "empty") {
+ t.Errorf("nil receiver = %q", got)
+ }
+ e2 := &propagationUnexpectedSkipErr{}
+ if got := e2.Error(); !strings.Contains(got, "empty") {
+ t.Errorf("empty slice = %q", got)
+ }
+}
+
+func TestPropagation_PickEligible_RowScanFails_Skipped(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ mock.ExpectBegin()
+ // One row with bad payload bytes for the uuid id → Scan fails → row
+ // skipped → empty result → COMMIT.
+ mock.ExpectQuery(`SELECT id, kind, team_id`).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "kind", "team_id", "target_tier", "payload", "attempts",
+ }).AddRow("not-a-uuid", "tier_elevation", "also-bad", nil, []byte(`{}`), 0))
+ mock.ExpectCommit()
+
+ w := NewPropagationRunnerWorker(db, nil, &stubPropagationRegrader{})
+ if err := w.Work(context.Background(), localJob[PropagationRunnerArgs]()); err != nil {
+ t.Fatalf("scan errors must not propagate: %v", err)
+ }
+}
+
+// ─── geodb.go: extractGeoLite2MMDB via in-package call ─────────────────────────
+
+func TestGeoDB_Extract_RoundTrip_DirectCall(t *testing.T) {
+ // Build a minimal valid tarball and call the unexported helper directly.
+ var buf bytes.Buffer
+ gz := gzip.NewWriter(&buf)
+ tw := tar.NewWriter(gz)
+ body := []byte("mmdb-bytes")
+ hdr := &tar.Header{
+ Name: "dir/GeoLite2-City.mmdb",
+ Typeflag: tar.TypeReg,
+ Size: int64(len(body)),
+ Mode: 0o644,
+ }
+ if err := tw.WriteHeader(hdr); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := tw.Write(body); err != nil {
+ t.Fatal(err)
+ }
+ tw.Close()
+ gz.Close()
+
+ dst := filepath.Join(t.TempDir(), "out.mmdb")
+ if err := extractGeoLite2MMDB(&buf, dst); err != nil {
+ t.Fatal(err)
+ }
+ got, err := os.ReadFile(dst)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(got, body) {
+ t.Errorf("mismatch: %q vs %q", got, body)
+ }
+}
+
+func TestGeoDB_Extract_BadGzip_Errors(t *testing.T) {
+ if err := extractGeoLite2MMDB(bytes.NewReader([]byte("not-gzip")), filepath.Join(t.TempDir(), "x")); err == nil {
+ t.Fatal("expected gzip error")
+ }
+}
+
+func TestGeoDB_Extract_NoMMDB_Errors(t *testing.T) {
+ var buf bytes.Buffer
+ gz := gzip.NewWriter(&buf)
+ tw := tar.NewWriter(gz)
+ hdr := &tar.Header{Name: "readme.txt", Typeflag: tar.TypeReg, Size: 3}
+ tw.WriteHeader(hdr)
+ tw.Write([]byte("abc"))
+ tw.Close()
+ gz.Close()
+ if err := extractGeoLite2MMDB(&buf, filepath.Join(t.TempDir(), "out")); err == nil {
+ t.Fatal("expected no-mmdb error")
+ }
+}
+
+// ─── resource_heartbeat.go: SampleDegradedGauge with rows ──────────────────────
+
+func TestHeartbeat_SampleGauge_PopulatedRowsAndResets(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ // Resource list returns zero (we want to reach the gauge sample only).
+ resID := uuid.New()
+ mock.ExpectQuery(`FROM resources`).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "token", "resource_type", "connection_url",
+ "team_id_text", "degraded", "last_seen_at",
+ }).AddRow(resID, uuid.New(), "postgres", "url", "", false, sql.NullTime{}))
+ mock.ExpectExec(`UPDATE resources\s+SET last_seen_at`).
+ WithArgs(resID).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ // gauge query: include one row with a bad type to cover the scan-error
+ // continue branch.
+ mock.ExpectQuery(`SELECT resource_type, COUNT\(\*\)`).
+ WillReturnRows(sqlmock.NewRows([]string{"resource_type", "count"}).
+ AddRow("postgres", "not-an-int").
+ AddRow("redis", int64(7)))
+
+ w := NewResourceHeartbeatWorker(db, &localFakeProber{outcome: ProbeReachable})
+ if err := w.Work(context.Background(), localJob[ResourceHeartbeatArgs]()); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestHeartbeat_RecoveryAuditInsertError_Logged(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ resID := uuid.New()
+ mock.ExpectQuery(`FROM resources`).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "token", "resource_type", "connection_url",
+ "team_id_text", "degraded", "last_seen_at",
+ }).AddRow(resID, uuid.New(), "redis", "url", "", true, sql.NullTime{Time: time.Now(), Valid: true}))
+ mock.ExpectExec(`UPDATE resources\s+SET last_seen_at`).
+ WithArgs(resID).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs(nil, "system", "resource.recovered", sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnError(errors.New("audit insert failed"))
+ mock.ExpectQuery(`SELECT resource_type, COUNT\(\*\)`).
+ WillReturnRows(sqlmock.NewRows([]string{"resource_type", "count"}))
+
+ w := NewResourceHeartbeatWorker(db, &localFakeProber{outcome: ProbeReachable})
+ if err := w.Work(context.Background(), localJob[ResourceHeartbeatArgs]()); err != nil {
+ t.Fatalf("fail-open: %v", err)
+ }
+}
+
+func TestHeartbeat_DegradedAuditInsertError_Logged(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ resID := uuid.New()
+ mock.ExpectQuery(`FROM resources`).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "token", "resource_type", "connection_url",
+ "team_id_text", "degraded", "last_seen_at",
+ }).AddRow(resID, uuid.New(), "postgres", "url", "", false, sql.NullTime{Time: time.Now().Add(-2 * time.Hour), Valid: true}))
+ mock.ExpectExec(`UPDATE resources\s+SET degraded = true`).
+ WithArgs(resID, sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(0, 1))
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs(nil, "system", "resource.degraded", sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnError(errors.New("audit insert failed"))
+ mock.ExpectQuery(`SELECT resource_type, COUNT\(\*\)`).
+ WillReturnRows(sqlmock.NewRows([]string{"resource_type", "count"}))
+
+ w := NewResourceHeartbeatWorker(db, &localFakeProber{outcome: ProbeUnreachable, err: errors.New("boom")})
+ if err := w.Work(context.Background(), localJob[ResourceHeartbeatArgs]()); err != nil {
+ t.Fatalf("fail-open: %v", err)
+ }
+}
+
+// ─── team_deletion_executor.go: processTeam + S3 deletion paths ───────────────
+
+// fakeS3Deleter fakes S3BackupDeleter for the executor's S3 step.
+type fakeS3Deleter struct {
+ listObjects []minio.ObjectInfo
+ listErr error
+ rmErrs []minio.RemoveObjectError
+}
+
+func (f *fakeS3Deleter) ListObjects(ctx context.Context, _ string, _ minio.ListObjectsOptions) <-chan minio.ObjectInfo {
+ ch := make(chan minio.ObjectInfo, len(f.listObjects)+1)
+ go func() {
+ defer close(ch)
+ for _, o := range f.listObjects {
+ select {
+ case ch <- o:
+ case <-ctx.Done():
+ return
+ }
+ }
+ if f.listErr != nil {
+ ch <- minio.ObjectInfo{Err: f.listErr}
+ }
+ }()
+ return ch
+}
+
+func (f *fakeS3Deleter) RemoveObjects(_ context.Context, _ string, in <-chan minio.ObjectInfo, _ minio.RemoveObjectsOptions) <-chan minio.RemoveObjectError {
+ out := make(chan minio.RemoveObjectError, len(f.rmErrs)+1)
+ go func() {
+ defer close(out)
+ // drain input
+ for range in {
+ }
+ for _, e := range f.rmErrs {
+ out <- e
+ }
+ }()
+ return out
+}
+
+func TestTeamDeletion_DeleteS3Backups_EmptyTokenNoOp(t *testing.T) {
+ db, _, _ := sqlmock.New()
+ defer db.Close()
+ w := NewTeamDeletionExecutorWorker(db, nil, &fakeS3Deleter{}, nil, "instant-shared")
+ got, err := w.deleteS3BackupsForToken(context.Background(), "")
+ if err != nil || got != 0 {
+ t.Errorf("empty token = (%d, %v)", got, err)
+ }
+}
+
+func TestTeamDeletion_DeleteS3Backups_HappyPath(t *testing.T) {
+ db, _, _ := sqlmock.New()
+ defer db.Close()
+ s3 := &fakeS3Deleter{
+ listObjects: []minio.ObjectInfo{
+ {Key: "backups/tok/a", Size: 100},
+ {Key: "backups/tok/b", Size: 250},
+ },
+ }
+ w := NewTeamDeletionExecutorWorker(db, nil, s3, nil, "instant-shared")
+ got, err := w.deleteS3BackupsForToken(context.Background(), uuid.New().String())
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got != 350 {
+ t.Errorf("bytes freed = %d, want 350", got)
+ }
+}
+
+func TestTeamDeletion_DeleteS3Backups_ListError(t *testing.T) {
+ db, _, _ := sqlmock.New()
+ defer db.Close()
+ s3 := &fakeS3Deleter{listErr: errors.New("list boom")}
+ w := NewTeamDeletionExecutorWorker(db, nil, s3, nil, "instant-shared")
+ _, err := w.deleteS3BackupsForToken(context.Background(), uuid.New().String())
+ if err == nil {
+ t.Fatal("expected error")
+ }
+}
+
+func TestTeamDeletion_DeleteS3Backups_RemoveError(t *testing.T) {
+ db, _, _ := sqlmock.New()
+ defer db.Close()
+ s3 := &fakeS3Deleter{
+ listObjects: []minio.ObjectInfo{{Key: "backups/x/a", Size: 1}},
+ rmErrs: []minio.RemoveObjectError{
+ {ObjectName: "backups/x/a", Err: errors.New("rm failed")},
+ },
+ }
+ w := NewTeamDeletionExecutorWorker(db, nil, s3, nil, "instant-shared")
+ _, err := w.deleteS3BackupsForToken(context.Background(), uuid.New().String())
+ if err == nil {
+ t.Fatal("expected error")
+ }
+}
+
+func TestTeamDeletion_ProcessTeam_MarkPendingError(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ teamID := uuid.New()
+ mock.ExpectQuery(`FROM teams\s+WHERE`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "deletion_requested_at"}).
+ AddRow(teamID, time.Now().Add(-90*24*time.Hour)))
+ mock.ExpectExec(`UPDATE teams\s+SET status = 'deletion_pending'`).
+ WithArgs(teamID).
+ WillReturnError(errors.New("flip failed"))
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs(teamID, "system", auditKindTeamDeletionFailed, sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+
+ w := NewTeamDeletionExecutorWorker(db, nil, nil, nil, "")
+ if err := w.Work(context.Background(), localJob[TeamDeletionExecutorArgs]()); err != nil {
+ t.Fatalf("per-team errors must be isolated: %v", err)
+ }
+ if err := mock.ExpectationsWereMet(); err != nil {
+ t.Errorf("unmet: %v", err)
+ }
+}
+
+func TestTeamDeletion_ProcessTeam_FetchResourcesError(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ teamID := uuid.New()
+ mock.ExpectQuery(`FROM teams\s+WHERE`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "deletion_requested_at"}).
+ AddRow(teamID, time.Now().Add(-90*24*time.Hour)))
+ mock.ExpectExec(`UPDATE teams\s+SET status = 'deletion_pending'`).
+ WithArgs(teamID).
+ WillReturnResult(sqlmock.NewResult(0, 1))
+ mock.ExpectQuery(`FROM resources\s+WHERE team_id`).
+ WithArgs(teamID).
+ WillReturnError(errors.New("res query failed"))
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs(teamID, "system", auditKindTeamDeletionFailed, sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+
+ w := NewTeamDeletionExecutorWorker(db, nil, nil, nil, "")
+ if err := w.Work(context.Background(), localJob[TeamDeletionExecutorArgs]()); err != nil {
+ t.Fatalf("per-team isolated: %v", err)
+ }
+}
+
+func TestTeamDeletion_ProcessTeam_S3Error(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ teamID := uuid.New()
+ resID := uuid.New()
+ mock.ExpectQuery(`FROM teams\s+WHERE`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "deletion_requested_at"}).
+ AddRow(teamID, time.Now().Add(-90*24*time.Hour)))
+ mock.ExpectExec(`UPDATE teams\s+SET status = 'deletion_pending'`).
+ WithArgs(teamID).
+ WillReturnResult(sqlmock.NewResult(0, 1))
+ mock.ExpectQuery(`FROM resources\s+WHERE team_id`).
+ WithArgs(teamID).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "token", "resource_type", "provider_resource_id"}).
+ AddRow(resID, uuid.New().String(), "postgres", ""))
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs(teamID, "system", auditKindTeamDeletionFailed, sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+
+ s3 := &fakeS3Deleter{listErr: errors.New("s3 boom")}
+ w := NewTeamDeletionExecutorWorker(db, nil, s3, nil, "instant-shared")
+ if err := w.Work(context.Background(), localJob[TeamDeletionExecutorArgs]()); err != nil {
+ t.Fatalf("per-team isolated: %v", err)
+ }
+}
+
+func TestTeamDeletion_ProcessTeam_K8sFetchAppIDsError(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ teamID := uuid.New()
+ mock.ExpectQuery(`FROM teams\s+WHERE`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "deletion_requested_at"}).
+ AddRow(teamID, time.Now().Add(-90*24*time.Hour)))
+ mock.ExpectExec(`UPDATE teams\s+SET status = 'deletion_pending'`).
+ WithArgs(teamID).
+ WillReturnResult(sqlmock.NewResult(0, 1))
+ mock.ExpectQuery(`FROM resources\s+WHERE team_id`).
+ WithArgs(teamID).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "token", "resource_type", "provider_resource_id"}))
+ mock.ExpectQuery(`SELECT DISTINCT app_id`).
+ WithArgs(teamID).
+ WillReturnError(errors.New("appid query failed"))
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs(teamID, "system", auditKindTeamDeletionFailed, sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+
+ k8s := newLocalFakeK8s()
+ w := NewTeamDeletionExecutorWorker(db, nil, nil, k8s, "")
+ if err := w.Work(context.Background(), localJob[TeamDeletionExecutorArgs]()); err != nil {
+ t.Fatalf("per-team isolated: %v", err)
+ }
+}
+
+func TestTeamDeletion_ProcessTeam_K8sDeleteNamespaceError(t *testing.T) {
+ db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ teamID := uuid.New()
+ mock.ExpectQuery(`FROM teams\s+WHERE`).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "deletion_requested_at"}).
+ AddRow(teamID, time.Now().Add(-90*24*time.Hour)))
+ mock.ExpectExec(`UPDATE teams\s+SET status = 'deletion_pending'`).
+ WithArgs(teamID).
+ WillReturnResult(sqlmock.NewResult(0, 1))
+ mock.ExpectQuery(`FROM resources\s+WHERE team_id`).
+ WithArgs(teamID).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "token", "resource_type", "provider_resource_id"}))
+ mock.ExpectQuery(`SELECT DISTINCT app_id`).
+ WithArgs(teamID).
+ WillReturnRows(sqlmock.NewRows([]string{"app_id"}).AddRow("appfoo"))
+ mock.ExpectExec(`INSERT INTO audit_log`).
+ WithArgs(teamID, "system", auditKindTeamDeletionFailed, sqlmock.AnyArg(), sqlmock.AnyArg()).
+ WillReturnResult(sqlmock.NewResult(1, 1))
+
+ k8s := newLocalFakeK8s()
+ k8s.failOn["instant-deploy-appfoo"] = errors.New("delete failed")
+ w := NewTeamDeletionExecutorWorker(db, nil, nil, k8s, "")
+ if err := w.Work(context.Background(), localJob[TeamDeletionExecutorArgs]()); err != nil {
+ t.Fatalf("per-team isolated: %v", err)
+ }
+}
+
+// localFakeK8s is an in-package K8sNamespaceDeleter double.
+type localFakeK8s struct {
+ mu sync.Mutex
+ deleted []string
+ failOn map[string]error
+}
+
+func newLocalFakeK8s() *localFakeK8s {
+ return &localFakeK8s{failOn: map[string]error{}}
+}
+
+func (f *localFakeK8s) DeleteNamespace(_ context.Context, ns string) error {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ if e, ok := f.failOn[ns]; ok {
+ return e
+ }
+ f.deleted = append(f.deleted, ns)
+ return nil
+}
+
+func (f *localFakeK8s) NamespaceExists(_ context.Context, _ string) (bool, error) {
+ return true, nil
+}
+
+// ─── real_prober.go: extra branches ───────────────────────────────────────────
+
+func TestProber_RealProber_DecryptGarbageReturnsErr(t *testing.T) {
+ p := NewRealProber(&config.Config{AESKey: "0000000000000000000000000000000000000000000000000000000000000000"}).(*realProber)
+ if _, err := p.decrypt("garbage-no-scheme"); err == nil {
+ t.Error("expected error for garbage without scheme")
+ }
+}
+
+// ─── compile-time guard ───────────────────────────────────────────────────────
+var _ = fmt.Sprintf
diff --git a/internal/jobs/geodb.go b/internal/jobs/geodb.go
index b2f99f1..2a40c7d 100644
--- a/internal/jobs/geodb.go
+++ b/internal/jobs/geodb.go
@@ -33,7 +33,11 @@ func NewRefreshGeoDBWorker() *RefreshGeoDBWorker {
return &RefreshGeoDBWorker{}
}
-const geoLite2DownloadURL = "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=%s&suffix=tar.gz"
+// geoLite2DownloadURL is the MaxMind GeoLite2-City download template (the
+// license key is interpolated in). It is a package var rather than a const
+// so the happy-path download→extract→rename pipeline in Work can be driven
+// against an httptest server in tests; production never reassigns it.
+var geoLite2DownloadURL = "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=%s&suffix=tar.gz"
// geoLite2MMDBSuffix is the file-name suffix of the MMDB member inside the
// MaxMind tarball. The archive contains a dated directory