From adad7e63c7774bb16f793614b62ae98494e9251a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 15 May 2026 10:26:13 +0000 Subject: [PATCH 1/3] fix(cli): skip pg_dump in db pull when using pg-delta diff engine Initial pulls under --diff-engine pg-delta were dumping the remote schema via pg_dump before running pg-delta. The dump-then-restore round-trip strips ownership information for objects the local postgres role cannot assume, so platform-managed objects (FDWs, wasm wrappers, system-owned ACLs) leak into the migration file and break \`supabase db reset\` (see CLI-1469, CLI-1470). pg-delta speaks pg_catalog directly via extractCatalog and the supabase integration filters platform objects by owner. Diffing against an empty shadow on initial pull yields a clean initial migration on its own, so dumpRemoteSchema is unnecessary on this path. --- apps/cli-go/internal/db/pull/pull.go | 23 +++++++++++++++---- apps/cli-go/internal/db/pull/pull_test.go | 28 +++++++++++++++++++++++ 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/apps/cli-go/internal/db/pull/pull.go b/apps/cli-go/internal/db/pull/pull.go index 210dcdfc9b..758cb74810 100644 --- a/apps/cli-go/internal/db/pull/pull.go +++ b/apps/cli-go/internal/db/pull/pull.go @@ -114,11 +114,20 @@ func run(ctx context.Context, schema []string, path string, conn *pgx.Conn, useP config := conn.Config().Config // 1. Assert `supabase/migrations` and `schema_migrations` are in sync. if err := assertRemoteInSync(ctx, conn, fsys); errors.Is(err, errMissing) { - // Ignore schemas flag when working on the initial pull - if err = dumpRemoteSchema(ctx, path, config, fsys); err != nil { - return err + // pg_dump strips ownership when restored as a non-superuser, so platform + // objects (FDWs, wasm wrappers, system-owned ACLs) leak into the migration + // and later break `supabase db reset`. pg-delta speaks pg_catalog directly + // and the supabase integration filter drops these by owner, so the diff + // against an empty shadow yields a clean initial migration on its own. + if !usePgDeltaDiff { + // Ignore schemas flag when working on the initial pull + if err = dumpRemoteSchema(ctx, path, config, fsys); err != nil { + return err + } } - // Run a second pass to pull in changes from default privileges and managed schemas + // For the legacy path this is a second pass that captures changes + // pg_dump cannot emit (default privileges, managed schemas). For the + // pg-delta path this is the only pass and produces the full schema. if err = diffRemoteSchema(ctx, nil, path, config, usePgDeltaDiff, differ, fsys); errors.Is(err, errInSync) { err = nil } @@ -153,7 +162,11 @@ func diffRemoteSchema(ctx context.Context, schema []string, path string, config if trimmed := strings.TrimSpace(output); len(trimmed) == 0 { return errors.New(errInSync) } - // Append to existing migration file since we run this after dump + if err := utils.MkdirIfNotExistFS(fsys, filepath.Dir(path)); err != nil { + return err + } + // Append to existing migration file when we run this after dumpRemoteSchema; + // for the pg-delta path this is the only writer and creates the file fresh. f, err := fsys.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) if err != nil { return errors.Errorf("failed to open migration file: %w", err) diff --git a/apps/cli-go/internal/db/pull/pull_test.go b/apps/cli-go/internal/db/pull/pull_test.go index 964a40dbe6..19a1161850 100644 --- a/apps/cli-go/internal/db/pull/pull_test.go +++ b/apps/cli-go/internal/db/pull/pull_test.go @@ -84,6 +84,34 @@ func TestPullSchema(t *testing.T) { assert.Equal(t, []byte("test"), contents) }) + t.Run("skips pg_dump for pg-delta diff engine on initial pull", func(t *testing.T) { + errNetwork := errors.New("network error") + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Setup mock docker. Only mock the image inspect call that + // CreateShadowDatabase makes; do NOT mock the pg_dump container so + // the test fails loudly if pg_dump is still invoked. + require.NoError(t, apitest.MockDocker(utils.Docker)) + defer gock.OffAll() + gock.New(utils.Docker.DaemonHost()). + Get("/v" + utils.Docker.ClientVersion() + "/images/" + utils.GetRegistryImageUrl(utils.Config.Db.Image) + "/json"). + ReplyError(errNetwork) + // Setup mock postgres (no local migrations -> initial pull path) + conn := pgtest.NewConn() + defer conn.Close(t) + conn.Query(migration.LIST_MIGRATION_VERSION). + Reply("SELECT 0") + // Run test with usePgDeltaDiff=true + err := run(context.Background(), nil, "0_test.sql", conn.MockClient(t), true, diff.DiffPgDelta, fsys) + // Failure must come from shadow-creation image inspect (proving we + // reached the diff step), not from pg_dump. + assert.ErrorIs(t, err, errNetwork) + assert.Empty(t, apitest.ListUnmatchedRequests()) + exists, err := afero.Exists(fsys, "0_test.sql") + assert.NoError(t, err) + assert.False(t, exists, "pg_dump should be skipped for pg-delta diff engine") + }) + t.Run("throws error on diff failure", func(t *testing.T) { // Setup in-memory fs fsys := afero.NewMemMapFs() From a51a612f2754db8fd75fb125bdd465cbbb6c7b80 Mon Sep 17 00:00:00 2001 From: avallete Date: Wed, 20 May 2026 13:34:44 +0200 Subject: [PATCH 2/3] wip --- .repos/effect | 2 +- .repos/effect-v3 | 2 +- .repos/lalph | 2 +- .repos/process-compose | 2 +- .repos/t3code | 2 +- apps/cli-go/cmd/db.go | 9 +- apps/cli-go/cmd/db_pull_routing_test.go | 29 +++++ apps/cli-go/docs/supabase/db/pull.md | 28 ++++- apps/cli-go/internal/db/declarative/debug.go | 32 +++-- apps/cli-go/internal/db/diff/diff.go | 44 +++++-- apps/cli-go/internal/db/diff/diff_test.go | 16 +-- apps/cli-go/internal/db/diff/pgdelta.go | 87 ++++++++----- apps/cli-go/internal/db/diff/pgdelta_debug.go | 85 +++++++++++++ .../internal/db/diff/pgdelta_debug_test.go | 50 ++++++++ .../internal/db/diff/templates/pgdelta.ts | 20 ++- apps/cli-go/internal/db/pgcache/cache.go | 7 +- .../internal/db/pull/pgdelta_pull_debug.go | 113 +++++++++++++++++ .../db/pull/pgdelta_pull_debug_test.go | 96 +++++++++++++++ apps/cli-go/internal/db/pull/pull.go | 44 +++++-- apps/cli-go/internal/db/pull/pull_test.go | 37 ++++++ .../cli-go/internal/gen/types/pgdelta_conn.go | 114 ++++++++++++++++++ .../internal/gen/types/pgdelta_conn_test.go | 53 ++++++++ apps/cli/package.json | 3 +- 23 files changed, 799 insertions(+), 78 deletions(-) create mode 100644 apps/cli-go/cmd/db_pull_routing_test.go create mode 100644 apps/cli-go/internal/db/diff/pgdelta_debug.go create mode 100644 apps/cli-go/internal/db/diff/pgdelta_debug_test.go create mode 100644 apps/cli-go/internal/db/pull/pgdelta_pull_debug.go create mode 100644 apps/cli-go/internal/db/pull/pgdelta_pull_debug_test.go create mode 100644 apps/cli-go/internal/gen/types/pgdelta_conn.go create mode 100644 apps/cli-go/internal/gen/types/pgdelta_conn_test.go diff --git a/.repos/effect b/.repos/effect index 49b5a569ea..aae8797b9c 160000 --- a/.repos/effect +++ b/.repos/effect @@ -1 +1 @@ -Subproject commit 49b5a569ea6c9d459f3db9cb2f150ca9d04b3cd0 +Subproject commit aae8797b9cb383be0c182dd58d03d787c354238b diff --git a/.repos/effect-v3 b/.repos/effect-v3 index e71ba68273..70ce155cd7 160000 --- a/.repos/effect-v3 +++ b/.repos/effect-v3 @@ -1 +1 @@ -Subproject commit e71ba68273026a1a2c1ace7218bdb206b0d3386d +Subproject commit 70ce155cd73a3b4cd723fe955454b5837b428f76 diff --git a/.repos/lalph b/.repos/lalph index 0fddf20a7d..203f1ec28f 160000 --- a/.repos/lalph +++ b/.repos/lalph @@ -1 +1 @@ -Subproject commit 0fddf20a7d70391bb583fb0abadc4223b618ec4d +Subproject commit 203f1ec28f26d3a4f18c0f3e092eae3695de1842 diff --git a/.repos/process-compose b/.repos/process-compose index a4038d6698..cd7f6af235 160000 --- a/.repos/process-compose +++ b/.repos/process-compose @@ -1 +1 @@ -Subproject commit a4038d669818c35fc68fc7fc240b39e371ce0e7a +Subproject commit cd7f6af235149a075385f3b8b54d635e83dc0f52 diff --git a/.repos/t3code b/.repos/t3code index d1e85c4e8f..91a03e0747 160000 --- a/.repos/t3code +++ b/.repos/t3code @@ -1 +1 @@ -Subproject commit d1e85c4e8fdef82fbaded9539532b754080419e0 +Subproject commit 91a03e074751e9dc732d0dddcd7b3a291caba34f diff --git a/apps/cli-go/cmd/db.go b/apps/cli-go/cmd/db.go index 99d1f81343..e4487deb3e 100644 --- a/apps/cli-go/cmd/db.go +++ b/apps/cli-go/cmd/db.go @@ -181,7 +181,7 @@ var ( if usePgDeltaDiff { pullDiffer = diff.DiffPgDelta } - useDeclarativePgDelta := shouldUsePgDelta() + useDeclarativePgDelta := shouldUseDeclarativePgDeltaPull(usePgDeltaDiff) return pull.Run(cmd.Context(), schema, flags.DbConfig, name, useDeclarativePgDelta, usePgDeltaDiff, pullDiffer, afero.NewOsFs()) }, PostRun: func(cmd *cobra.Command, args []string) { @@ -358,6 +358,13 @@ func shouldUsePgDelta() bool { return utils.IsPgDeltaEnabled() || usePgDelta || viper.GetBool("EXPERIMENTAL_PG_DELTA") } +func shouldUseDeclarativePgDeltaPull(usePgDeltaDiff bool) bool { + if usePgDeltaDiff { + return false + } + return shouldUsePgDelta() || usePgDelta +} + func init() { // Build branch command dbBranchCmd.AddCommand(dbBranchCreateCmd) diff --git a/apps/cli-go/cmd/db_pull_routing_test.go b/apps/cli-go/cmd/db_pull_routing_test.go new file mode 100644 index 0000000000..8a01e782f8 --- /dev/null +++ b/apps/cli-go/cmd/db_pull_routing_test.go @@ -0,0 +1,29 @@ +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestShouldUseDeclarativePgDeltaPull(t *testing.T) { + t.Run("migration pg-delta wins over experimental config", func(t *testing.T) { + usePgDelta = false + t.Cleanup(func() { usePgDelta = false }) + assert.False(t, shouldUseDeclarativePgDeltaPull(true)) + }) + + t.Run("experimental config without diff-engine uses declarative", func(t *testing.T) { + usePgDelta = false + t.Cleanup(func() { usePgDelta = false }) + // Simulate config enabled via shouldUsePgDelta's IsPgDeltaEnabled path indirectly: + // when neither flag nor config is set, declarative is off. + assert.False(t, shouldUseDeclarativePgDeltaPull(false)) + }) + + t.Run("use-pg-delta flag forces declarative", func(t *testing.T) { + usePgDelta = true + t.Cleanup(func() { usePgDelta = false }) + assert.True(t, shouldUseDeclarativePgDeltaPull(false)) + }) +} diff --git a/apps/cli-go/docs/supabase/db/pull.md b/apps/cli-go/docs/supabase/db/pull.md index b137aaa42a..93f964b9a9 100644 --- a/apps/cli-go/docs/supabase/db/pull.md +++ b/apps/cli-go/docs/supabase/db/pull.md @@ -8,6 +8,30 @@ Requires your local project to be linked to a remote database by running `supaba Optionally, a new row can be inserted into the migration history table to reflect the current state of the remote database. -If no entries exist in the migration history table, `pg_dump` will be used to capture all contents of the remote schemas you have created. Otherwise, this command will only diff schema changes against the remote database, similar to running `db diff --linked`. +If no entries exist in the migration history table, the default diff engine uses `pg_dump` to capture all contents of the remote schemas you have created. Otherwise, this command will only diff schema changes against the remote database, similar to running `db diff --linked`. -Pass `--diff-engine pg-delta` to keep the migration-file `db pull` workflow while using pg-delta for the shadow diff step. Pass `--use-pg-delta` to switch to the declarative pg-delta export workflow instead. +Pass `--diff-engine pg-delta` to keep the migration-file `db pull` workflow while using pg-delta for the shadow diff step. On initial pull, pg-delta replaces `pg_dump` and produces the full migration from the shadow diff alone. Pass `--use-pg-delta` to switch to the declarative pg-delta export workflow instead. + +When `[experimental.pgdelta] enabled = true` is set in `config.toml`, `db pull` defaults to the declarative export path. Explicit `--diff-engine pg-delta` still selects the migration-file workflow. + +When pulling from a remote database with `--db-url`, prefer a direct connection (`db..supabase.co:5432`) over the connection pooler so pg-delta can introspect the full catalog reliably. + +## Debugging empty pg-delta pulls + +If `db pull --diff-engine pg-delta` reports `No schema changes found` but you expect schema output, set `PGDELTA_DEBUG=1` before running the command. Unlike `--debug`, this keeps SSL enabled for remote Supabase connections. + +```sh +PGDELTA_DEBUG=1 supabase db pull --db-url "$DATABASE_URL" --diff-engine pg-delta +``` + +When pg-delta returns zero statements, the CLI writes a debug bundle under `supabase/.temp/pgdelta/debug//`: + +- `source-catalog.json` — shadow database baseline pg-delta extracted +- `target-catalog.json` — remote database pg-delta extracted +- `pgdelta-stderr.txt` — pg-delta script diagnostics (statement count, schemas) +- `connection.txt` — redacted connection metadata +- `error.txt` — error summary + +Catalog files are not written during normal `db pull` runs. The `.temp/pgdelta` directory is also used by migration catalog caching (`db push`, local `db start`) when `[experimental.pgdelta] enabled = true`. + +For TLS tracing without disabling SSL, use `SUPABASE_SSL_DEBUG=true` alongside `PGDELTA_DEBUG=1`. diff --git a/apps/cli-go/internal/db/declarative/debug.go b/apps/cli-go/internal/db/declarative/debug.go index f274ed39dd..779514c525 100644 --- a/apps/cli-go/internal/db/declarative/debug.go +++ b/apps/cli-go/internal/db/declarative/debug.go @@ -18,12 +18,16 @@ const ( // DebugBundle collects diagnostic artifacts when a declarative operation fails. type DebugBundle struct { - ID string // timestamp-based unique ID (e.g. "20240414-044403") - SourceRef string // path to source catalog - TargetRef string // path to target catalog - MigrationSQL string // generated migration (if available) - Error error // the error that occurred - Migrations []string // list of local migration files + ID string // timestamp-based unique ID (e.g. "20240414-044403") + SourceRef string // path to source catalog + TargetRef string // path to target catalog + SourceCatalog string // inline source catalog JSON (optional) + TargetCatalog string // inline target catalog JSON (optional) + MigrationSQL string // generated migration (if available) + PgDeltaStderr string // edge-runtime stderr from pg-delta scripts + ConnectionInfo string // redacted connection metadata + Error error // the error that occurred + Migrations []string // list of local migration files } // SaveDebugBundle writes diagnostic artifacts to .temp/pgdelta/debug// and @@ -38,14 +42,18 @@ func SaveDebugBundle(bundle DebugBundle, fsys afero.Fs) (string, error) { } // Copy source catalog if available - if len(bundle.SourceRef) > 0 { + if len(bundle.SourceCatalog) > 0 { + _ = utils.WriteFile(filepath.Join(debugDir, "source-catalog.json"), []byte(bundle.SourceCatalog), fsys) + } else if len(bundle.SourceRef) > 0 { if data, err := afero.ReadFile(fsys, bundle.SourceRef); err == nil { _ = utils.WriteFile(filepath.Join(debugDir, "source-catalog.json"), data, fsys) } } // Copy target catalog if available - if len(bundle.TargetRef) > 0 { + if len(bundle.TargetCatalog) > 0 { + _ = utils.WriteFile(filepath.Join(debugDir, "target-catalog.json"), []byte(bundle.TargetCatalog), fsys) + } else if len(bundle.TargetRef) > 0 { if data, err := afero.ReadFile(fsys, bundle.TargetRef); err == nil { _ = utils.WriteFile(filepath.Join(debugDir, "target-catalog.json"), data, fsys) } @@ -61,6 +69,14 @@ func SaveDebugBundle(bundle DebugBundle, fsys afero.Fs) (string, error) { _ = utils.WriteFile(filepath.Join(debugDir, "error.txt"), []byte(bundle.Error.Error()), fsys) } + if len(bundle.PgDeltaStderr) > 0 { + _ = utils.WriteFile(filepath.Join(debugDir, "pgdelta-stderr.txt"), []byte(bundle.PgDeltaStderr), fsys) + } + + if len(bundle.ConnectionInfo) > 0 { + _ = utils.WriteFile(filepath.Join(debugDir, "connection.txt"), []byte(bundle.ConnectionInfo), fsys) + } + // Copy migration files if len(bundle.Migrations) > 0 { migrationsDir := filepath.Join(debugDir, "migrations") diff --git a/apps/cli-go/internal/db/diff/diff.go b/apps/cli-go/internal/db/diff/diff.go index 05991423d6..1bca9bf39e 100644 --- a/apps/cli-go/internal/db/diff/diff.go +++ b/apps/cli-go/internal/db/diff/diff.go @@ -33,10 +33,11 @@ import ( type DiffFunc func(context.Context, pgconn.Config, pgconn.Config, []string, ...func(*pgx.ConnConfig)) (string, error) func Run(ctx context.Context, schema []string, file string, config pgconn.Config, differ DiffFunc, usePgDelta bool, fsys afero.Fs, options ...func(*pgx.ConnConfig)) (err error) { - out, err := DiffDatabase(ctx, schema, config, os.Stderr, fsys, differ, usePgDelta, options...) + result, err := DiffDatabase(ctx, schema, config, os.Stderr, fsys, differ, usePgDelta, options...) if err != nil { return err } + out := result.SQL branch := utils.GetGitBranch(fsys) fmt.Fprintln(os.Stderr, "Finished "+utils.Aqua("supabase db diff")+" on branch "+utils.Aqua(branch)+".\n") if err := SaveDiff(out, file, fsys); err != nil { @@ -161,18 +162,18 @@ func MigrateShadowDatabase(ctx context.Context, container string, fsys afero.Fs, return migration.ApplyMigrations(ctx, migrations, conn, afero.NewIOFS(fsys)) } -func DiffDatabase(ctx context.Context, schema []string, config pgconn.Config, w io.Writer, fsys afero.Fs, differ DiffFunc, usePgDelta bool, options ...func(*pgx.ConnConfig)) (string, error) { +func DiffDatabase(ctx context.Context, schema []string, config pgconn.Config, w io.Writer, fsys afero.Fs, differ DiffFunc, usePgDelta bool, options ...func(*pgx.ConnConfig)) (DatabaseDiff, error) { fmt.Fprintln(w, "Creating shadow database...") shadow, err := CreateShadowDatabase(ctx, utils.Config.Db.ShadowPort) if err != nil { - return "", err + return DatabaseDiff{}, err } defer utils.DockerRemove(shadow) if err := start.WaitForHealthyService(ctx, utils.Config.Db.HealthTimeout, shadow); err != nil { - return "", err + return DatabaseDiff{}, err } if err := MigrateShadowDatabase(ctx, shadow, fsys, options...); err != nil { - return "", err + return DatabaseDiff{}, err } shadowConfig := pgconn.Config{ Host: utils.Config.Hostname, @@ -189,20 +190,20 @@ func DiffDatabase(ctx context.Context, schema []string, config pgconn.Config, w declDir := utils.GetDeclarativeDir() if exists, _ := afero.DirExists(fsys, declDir); exists { if err := pgdelta.ApplyDeclarative(ctx, config, fsys); err != nil { - return "", err + return DatabaseDiff{}, err } } else { if err := migrateBaseDatabase(ctx, config, declared, fsys, options...); err != nil { - return "", err + return DatabaseDiff{}, err } } } else { if err := migrateBaseDatabase(ctx, config, declared, fsys, options...); err != nil { - return "", err + return DatabaseDiff{}, err } } } else if err != nil { - return "", err + return DatabaseDiff{}, err } } // Load all user defined schemas @@ -211,7 +212,30 @@ func DiffDatabase(ctx context.Context, schema []string, config pgconn.Config, w } else { fmt.Fprintln(w, "Diffing schemas...") } - return differ(ctx, shadowConfig, config, schema, options...) + var debugCapture *PgDeltaDebugCapture + if IsPgDeltaDebugEnabled() && usePgDelta { + if snapshot, exportErr := exportCatalogPgDelta(ctx, utils.ToPostgresURL(shadowConfig), "postgres", options...); exportErr == nil { + debugCapture = &PgDeltaDebugCapture{SourceCatalog: snapshot} + } else { + fmt.Fprintf(w, "Warning: failed to export shadow pg-delta catalog: %v\n", exportErr) + } + } + if IsPgDeltaDebugEnabled() && usePgDelta { + result, err := DiffPgDeltaRefDetailed(ctx, utils.ToPostgresURL(shadowConfig), utils.ToPostgresURL(config), schema, pgDeltaFormatOptions(), options...) + if err != nil { + return DatabaseDiff{}, err + } + if debugCapture == nil { + debugCapture = &PgDeltaDebugCapture{} + } + debugCapture.Stderr = result.Stderr + return DatabaseDiff{SQL: result.SQL, Debug: debugCapture}, nil + } + output, err := differ(ctx, shadowConfig, config, schema, options...) + if err != nil { + return DatabaseDiff{}, err + } + return DatabaseDiff{SQL: output, Debug: debugCapture}, nil } func migrateBaseDatabase(ctx context.Context, config pgconn.Config, migrations []string, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { diff --git a/apps/cli-go/internal/db/diff/diff_test.go b/apps/cli-go/internal/db/diff/diff_test.go index 363d7b3f5b..f14c6a9f73 100644 --- a/apps/cli-go/internal/db/diff/diff_test.go +++ b/apps/cli-go/internal/db/diff/diff_test.go @@ -203,9 +203,9 @@ func TestDiffDatabase(t *testing.T) { Get("/v" + utils.Docker.ClientVersion() + "/images/" + utils.GetRegistryImageUrl(utils.Config.Db.Image) + "/json"). ReplyError(errNetwork) // Run test - diff, err := DiffDatabase(context.Background(), []string{"public"}, dbConfig, io.Discard, fsys, DiffSchemaMigra, false) + result, err := DiffDatabase(context.Background(), []string{"public"}, dbConfig, io.Discard, fsys, DiffSchemaMigra, false) // Check error - assert.Empty(t, diff) + assert.Empty(t, result.SQL) assert.ErrorIs(t, err, errNetwork) assert.Empty(t, apitest.ListUnmatchedRequests()) }) @@ -234,9 +234,9 @@ func TestDiffDatabase(t *testing.T) { Delete("/v" + utils.Docker.ClientVersion() + "/containers/test-shadow-db"). Reply(http.StatusOK) // Run test - diff, err := DiffDatabase(context.Background(), []string{"public"}, dbConfig, io.Discard, fsys, DiffSchemaMigra, false) + result, err := DiffDatabase(context.Background(), []string{"public"}, dbConfig, io.Discard, fsys, DiffSchemaMigra, false) // Check error - assert.Empty(t, diff) + assert.Empty(t, result.SQL) assert.ErrorContains(t, err, "test-shadow-db container is not running: exited") assert.Empty(t, apitest.ListUnmatchedRequests()) }) @@ -266,9 +266,9 @@ func TestDiffDatabase(t *testing.T) { conn.Query(utils.GlobalsSql). ReplyError(pgerrcode.DuplicateSchema, `schema "public" already exists`) // Run test - diff, err := DiffDatabase(context.Background(), []string{"public"}, dbConfig, io.Discard, fsys, DiffSchemaMigra, false, conn.Intercept) + result, err := DiffDatabase(context.Background(), []string{"public"}, dbConfig, io.Discard, fsys, DiffSchemaMigra, false, conn.Intercept) // Check error - assert.Empty(t, diff) + assert.Empty(t, result.SQL) assert.ErrorContains(t, err, `ERROR: schema "public" already exists (SQLSTATE 42P06) At statement: 0 create schema public`) @@ -321,7 +321,7 @@ create schema public`) Query(migration.INSERT_MIGRATION_VERSION, "0", "test", []string{sql}). Reply("INSERT 0 1") // Run test - diff, err := DiffDatabase(context.Background(), []string{"public"}, dbConfig, io.Discard, fsys, DiffSchemaMigra, false, func(cc *pgx.ConnConfig) { + result, err := DiffDatabase(context.Background(), []string{"public"}, dbConfig, io.Discard, fsys, DiffSchemaMigra, false, func(cc *pgx.ConnConfig) { if cc.Host == dbConfig.Host { // Fake a SSL error when connecting to target database cc.LookupFunc = func(ctx context.Context, host string) (addrs []string, err error) { @@ -333,7 +333,7 @@ create schema public`) } }) // Check error - assert.Empty(t, diff) + assert.Empty(t, result.SQL) assert.ErrorContains(t, err, "error diffing schema") assert.Empty(t, apitest.ListUnmatchedRequests()) }) diff --git a/apps/cli-go/internal/db/diff/pgdelta.go b/apps/cli-go/internal/db/diff/pgdelta.go index dd6af84924..787e56cb33 100644 --- a/apps/cli-go/internal/db/diff/pgdelta.go +++ b/apps/cli-go/internal/db/diff/pgdelta.go @@ -68,6 +68,22 @@ func pgDeltaFormatOptions() string { return strings.TrimSpace(utils.Config.Experimental.PgDelta.FormatOptions) } +func appendPgDeltaPostgresEnv( + ctx context.Context, + env []string, + name string, + ref string, + sslRootCertEnv string, + options ...func(*pgx.ConnConfig), +) (string, []string, error) { + preparedRef, sslEnv, err := types.PreparePgDeltaPostgresRef(ctx, ref, sslRootCertEnv, options...) + if err != nil { + return "", nil, err + } + env = append(env, name+"="+containerRef(preparedRef)) + return preparedRef, append(env, sslEnv...), nil +} + // DiffPgDelta diffs source and target Postgres configs via pg-delta. // // This wrapper preserves the old config-based interface while delegating to @@ -81,17 +97,25 @@ func DiffPgDelta(ctx context.Context, source, target pgconn.Config, schema []str // on-disk catalog references used by declarative sync commands. formatOptions // is passed through as FORMAT_OPTIONS to the pg-delta script when non-empty. func DiffPgDeltaRef(ctx context.Context, sourceRef, targetRef string, schema []string, formatOptions string, options ...func(*pgx.ConnConfig)) (string, error) { - env := []string{ - "TARGET=" + containerRef(targetRef), + result, err := DiffPgDeltaRefDetailed(ctx, sourceRef, targetRef, schema, formatOptions, options...) + if err != nil { + return "", err } - if len(sourceRef) > 0 { - env = append(env, "SOURCE="+containerRef(sourceRef)) + return result.SQL, nil +} + +// DiffPgDeltaRefDetailed is like DiffPgDeltaRef but also returns edge-runtime stderr. +func DiffPgDeltaRefDetailed(ctx context.Context, sourceRef, targetRef string, schema []string, formatOptions string, options ...func(*pgx.ConnConfig)) (PgDeltaDiffResult, error) { + var env []string + var err error + targetRef, env, err = appendPgDeltaPostgresEnv(ctx, env, "TARGET", targetRef, types.PgDeltaTargetSSLRootCert, options...) + if err != nil { + return PgDeltaDiffResult{}, err } - if isPostgresURL(targetRef) { - if ca, err := types.GetRootCA(ctx, targetRef, options...); err != nil { - return "", err - } else if len(ca) > 0 { - env = append(env, "PGDELTA_TARGET_SSLROOTCERT="+ca) + if len(sourceRef) > 0 { + sourceRef, env, err = appendPgDeltaPostgresEnv(ctx, env, "SOURCE", sourceRef, types.PgDeltaSourceSSLRootCert, options...) + if err != nil { + return PgDeltaDiffResult{}, err } } if len(schema) > 0 { @@ -100,6 +124,9 @@ func DiffPgDeltaRef(ctx context.Context, sourceRef, targetRef string, schema []s if len(strings.TrimSpace(formatOptions)) > 0 { env = append(env, "FORMAT_OPTIONS="+formatOptions) } + if IsPgDeltaDebugEnabled() { + env = append(env, "PGDELTA_DEBUG=1") + } binds := []string{utils.EdgeRuntimeId + ":/root/.cache/deno:rw"} if cwd, err := os.Getwd(); err == nil { binds = append(binds, cwd+":/workspace") @@ -107,11 +134,17 @@ func DiffPgDeltaRef(ctx context.Context, sourceRef, targetRef string, schema []s var stdout, stderr bytes.Buffer script := config.InterpolatePgDeltaScript(config.Config(&utils.Config), pgDeltaScript) if err := utils.RunEdgeRuntimeScript(ctx, env, script, binds, "error diffing schema", &stdout, &stderr); err != nil { - return "", err + return PgDeltaDiffResult{}, err } - return stdout.String(), nil + return PgDeltaDiffResult{ + SQL: stdout.String(), + Stderr: stderr.String(), + }, nil } +// exportCatalogPgDelta is overridden in tests to mock catalog export. +var exportCatalogPgDelta = ExportCatalogPgDelta + // DeclarativeExportPgDelta exports target schema as declarative file payloads // while keeping a config-based API for existing call sites. func DeclarativeExportPgDelta(ctx context.Context, source, target pgconn.Config, schema []string, formatOptions string, options ...func(*pgx.ConnConfig)) (DeclarativeOutput, error) { @@ -121,17 +154,16 @@ func DeclarativeExportPgDelta(ctx context.Context, source, target pgconn.Config, // DeclarativeExportPgDeltaRef exports declarative file payloads using either // live URLs or catalog references as source/target inputs. func DeclarativeExportPgDeltaRef(ctx context.Context, sourceRef, targetRef string, schema []string, formatOptions string, options ...func(*pgx.ConnConfig)) (DeclarativeOutput, error) { - env := []string{ - "TARGET=" + containerRef(targetRef), + var env []string + var err error + targetRef, env, err = appendPgDeltaPostgresEnv(ctx, env, "TARGET", targetRef, types.PgDeltaTargetSSLRootCert, options...) + if err != nil { + return DeclarativeOutput{}, err } if len(sourceRef) > 0 { - env = append(env, "SOURCE="+containerRef(sourceRef)) - } - if isPostgresURL(targetRef) { - if ca, err := types.GetRootCA(ctx, targetRef, options...); err != nil { + sourceRef, env, err = appendPgDeltaPostgresEnv(ctx, env, "SOURCE", sourceRef, types.PgDeltaSourceSSLRootCert, options...) + if err != nil { return DeclarativeOutput{}, err - } else if len(ca) > 0 { - env = append(env, "PGDELTA_TARGET_SSLROOTCERT="+ca) } } if len(schema) > 0 { @@ -140,6 +172,9 @@ func DeclarativeExportPgDeltaRef(ctx context.Context, sourceRef, targetRef strin if len(strings.TrimSpace(formatOptions)) > 0 { env = append(env, "FORMAT_OPTIONS="+formatOptions) } + if IsPgDeltaDebugEnabled() { + env = append(env, "PGDELTA_DEBUG=1") + } binds := []string{utils.EdgeRuntimeId + ":/root/.cache/deno:rw"} if cwd, err := os.Getwd(); err == nil { binds = append(binds, cwd+":/workspace") @@ -162,19 +197,15 @@ func DeclarativeExportPgDeltaRef(ctx context.Context, sourceRef, targetRef strin // ExportCatalogPgDelta snapshots a database/catalog into serialized pg-delta // catalog JSON so later operations can diff without reconnecting. func ExportCatalogPgDelta(ctx context.Context, targetRef, role string, options ...func(*pgx.ConnConfig)) (string, error) { - env := []string{ - "TARGET=" + targetRef, + var env []string + var err error + targetRef, env, err = appendPgDeltaPostgresEnv(ctx, env, "TARGET", targetRef, types.PgDeltaTargetSSLRootCert, options...) + if err != nil { + return "", err } if len(role) > 0 { env = append(env, "ROLE="+role) } - if isPostgresURL(targetRef) { - if ca, err := types.GetRootCA(ctx, targetRef, options...); err != nil { - return "", err - } else if len(ca) > 0 { - env = append(env, "PGDELTA_TARGET_SSLROOTCERT="+ca) - } - } binds := []string{ utils.EdgeRuntimeId + ":/root/.cache/deno:rw", } diff --git a/apps/cli-go/internal/db/diff/pgdelta_debug.go b/apps/cli-go/internal/db/diff/pgdelta_debug.go new file mode 100644 index 0000000000..b901edd5f7 --- /dev/null +++ b/apps/cli-go/internal/db/diff/pgdelta_debug.go @@ -0,0 +1,85 @@ +package diff + +import ( + "encoding/json" + "os" + "strings" +) + +// IsPgDeltaDebugEnabled reports whether pg-delta diagnostic output is requested. +// Unlike --debug, this does not disable SSL for remote Postgres connections. +func IsPgDeltaDebugEnabled() bool { + switch strings.ToLower(strings.TrimSpace(os.Getenv("PGDELTA_DEBUG"))) { + case "1", "true", "yes": + return true + default: + return false + } +} + +// PgDeltaDiffResult holds pg-delta diff output and edge-runtime stderr. +type PgDeltaDiffResult struct { + SQL string + Stderr string +} + +// PgDeltaDebugCapture holds artifacts collected during a pg-delta shadow diff. +type PgDeltaDebugCapture struct { + SourceCatalog string + Stderr string +} + +// DatabaseDiff is the result of diffing a target database against a shadow baseline. +type DatabaseDiff struct { + SQL string + Debug *PgDeltaDebugCapture +} + +// CatalogSummary summarizes object counts extracted from a pg-delta catalog JSON blob. +type CatalogSummary struct { + TotalObjects int + BySchema map[string]int +} + +// SummarizeCatalogJSON best-effort counts catalog objects grouped by schema name. +func SummarizeCatalogJSON(catalogJSON string) CatalogSummary { + summary := CatalogSummary{BySchema: map[string]int{}} + if len(strings.TrimSpace(catalogJSON)) == 0 { + return summary + } + var root any + if err := json.Unmarshal([]byte(catalogJSON), &root); err != nil { + return summary + } + walkCatalogObjects(root, summary.BySchema, &summary.TotalObjects) + return summary +} + +func walkCatalogObjects(node any, bySchema map[string]int, total *int) { + switch value := node.(type) { + case map[string]any: + if schema, ok := schemaNameFromCatalogNode(value); ok { + *total++ + bySchema[schema]++ + } + for _, child := range value { + walkCatalogObjects(child, bySchema, total) + } + case []any: + for _, child := range value { + walkCatalogObjects(child, bySchema, total) + } + } +} + +func schemaNameFromCatalogNode(node map[string]any) (string, bool) { + if schema, ok := node["schema"].(string); ok && len(schema) > 0 { + return schema, true + } + if schemaObj, ok := node["schema"].(map[string]any); ok { + if name, ok := schemaObj["name"].(string); ok && len(name) > 0 { + return name, true + } + } + return "", false +} diff --git a/apps/cli-go/internal/db/diff/pgdelta_debug_test.go b/apps/cli-go/internal/db/diff/pgdelta_debug_test.go new file mode 100644 index 0000000000..a8c0d47763 --- /dev/null +++ b/apps/cli-go/internal/db/diff/pgdelta_debug_test.go @@ -0,0 +1,50 @@ +package diff + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsPgDeltaDebugEnabled(t *testing.T) { + t.Run("disabled by default", func(t *testing.T) { + t.Setenv("PGDELTA_DEBUG", "") + assert.False(t, IsPgDeltaDebugEnabled()) + }) + + t.Run("enabled for 1", func(t *testing.T) { + t.Setenv("PGDELTA_DEBUG", "1") + assert.True(t, IsPgDeltaDebugEnabled()) + }) + + t.Run("enabled for true", func(t *testing.T) { + t.Setenv("PGDELTA_DEBUG", "true") + assert.True(t, IsPgDeltaDebugEnabled()) + }) + + t.Run("enabled for yes", func(t *testing.T) { + t.Setenv("PGDELTA_DEBUG", "YES") + assert.True(t, IsPgDeltaDebugEnabled()) + }) +} + +func TestSummarizeCatalogJSON(t *testing.T) { + t.Run("counts schema objects", func(t *testing.T) { + catalog := `{ + "schemas": [ + {"schema": "public", "tables": [{"schema": "public", "name": "airports"}]}, + {"schema": "auth", "tables": [{"schema": "auth", "name": "users"}]} + ] + }` + summary := SummarizeCatalogJSON(catalog) + assert.Equal(t, 4, summary.TotalObjects) + assert.Equal(t, 2, summary.BySchema["public"]) + assert.Equal(t, 2, summary.BySchema["auth"]) + }) + + t.Run("returns empty summary for invalid json", func(t *testing.T) { + summary := SummarizeCatalogJSON("{not-json") + assert.Equal(t, 0, summary.TotalObjects) + assert.Empty(t, summary.BySchema) + }) +} diff --git a/apps/cli-go/internal/db/diff/templates/pgdelta.ts b/apps/cli-go/internal/db/diff/templates/pgdelta.ts index cb5359566b..cccd1bc95b 100644 --- a/apps/cli-go/internal/db/diff/templates/pgdelta.ts +++ b/apps/cli-go/internal/db/diff/templates/pgdelta.ts @@ -2,8 +2,8 @@ import { createPlan, deserializeCatalog, formatSqlStatements, -} from "npm:@supabase/pg-delta@1.0.0-alpha.20"; -import { supabase } from "npm:@supabase/pg-delta@1.0.0-alpha.20/integrations/supabase"; +} from "npm:@supabase/pg-delta@1.0.0-alpha.25"; +import { supabase } from "npm:@supabase/pg-delta@1.0.0-alpha.25/integrations/supabase"; async function resolveInput(ref: string | undefined) { if (!ref) { @@ -41,12 +41,26 @@ try { const result = await createPlan( await resolveInput(source), await resolveInput(target), - supabase, + { + ...supabase, + skipDefaultPrivilegeSubtraction: true, + }, ); let statements = result?.plan.statements ?? []; if (formatOptions != null) { statements = formatSqlStatements(statements, formatOptions); } + if (Deno.env.get("PGDELTA_DEBUG")) { + console.error( + JSON.stringify({ + statementCount: statements.length, + source: source ? "connected" : "null", + target: target ? "connected" : "null", + includedSchemas: includedSchemas ?? null, + skipDefaultPrivilegeSubtraction: true, + }), + ); + } for (const sql of statements) { console.log(`${sql};`); } diff --git a/apps/cli-go/internal/db/pgcache/cache.go b/apps/cli-go/internal/db/pgcache/cache.go index aeb1ebfbce..ea9bb202aa 100644 --- a/apps/cli-go/internal/db/pgcache/cache.go +++ b/apps/cli-go/internal/db/pgcache/cache.go @@ -246,12 +246,11 @@ func pgDeltaTempPath() string { } func exportCatalog(ctx context.Context, targetRef string, options ...func(*pgx.ConnConfig)) (string, error) { - env := []string{"TARGET=" + targetRef, "ROLE=postgres"} - if ca, err := types.GetRootCA(ctx, targetRef, options...); err != nil { + preparedRef, sslEnv, err := types.PreparePgDeltaPostgresRef(ctx, targetRef, types.PgDeltaTargetSSLRootCert, options...) + if err != nil { return "", err - } else if len(ca) > 0 { - env = append(env, "PGDELTA_TARGET_SSLROOTCERT="+ca) } + env := append([]string{"TARGET=" + preparedRef, "ROLE=postgres"}, sslEnv...) binds := []string{utils.EdgeRuntimeId + ":/root/.cache/deno:rw"} var stdout, stderr bytes.Buffer script := config.InterpolatePgDeltaScript(config.Config(&utils.Config), pgDeltaCatalogExportTS) diff --git a/apps/cli-go/internal/db/pull/pgdelta_pull_debug.go b/apps/cli-go/internal/db/pull/pgdelta_pull_debug.go new file mode 100644 index 0000000000..10881cc939 --- /dev/null +++ b/apps/cli-go/internal/db/pull/pgdelta_pull_debug.go @@ -0,0 +1,113 @@ +package pull + +import ( + "context" + "fmt" + "net/url" + "os" + "strings" + + "github.com/go-errors/errors" + "github.com/jackc/pgconn" + "github.com/jackc/pgx/v4" + "github.com/spf13/afero" + "github.com/supabase/cli/internal/db/declarative" + "github.com/supabase/cli/internal/db/diff" + "github.com/supabase/cli/internal/utils" +) + +var exportCatalogPgDelta = diff.ExportCatalogPgDelta + +func saveEmptyPgDeltaPullDebug( + ctx context.Context, + config pgconn.Config, + capture *diff.PgDeltaDebugCapture, + fsys afero.Fs, + options ...func(*pgx.ConnConfig), +) (string, error) { + if capture == nil { + capture = &diff.PgDeltaDebugCapture{} + } + targetCatalog, err := exportCatalogPgDelta(ctx, utils.ToPostgresURL(config), "postgres", options...) + if err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to export remote pg-delta catalog: %v\n", err) + } + bundle := declarative.DebugBundle{ + SourceCatalog: capture.SourceCatalog, + TargetCatalog: targetCatalog, + PgDeltaStderr: capture.Stderr, + ConnectionInfo: formatConnectionInfo(config), + Error: errors.New(errInSync), + } + debugDir, err := declarative.SaveDebugBundle(bundle, fsys) + if err != nil { + return "", err + } + printEmptyPgDeltaPullSummary(debugDir, capture.SourceCatalog, targetCatalog) + declarative.PrintDebugBundleMessage(debugDir) + return debugDir, nil +} + +func printEmptyPgDeltaPullSummary(debugDir, sourceCatalog, targetCatalog string) { + fmt.Fprintln(os.Stderr, "pg-delta returned 0 statements.") + fmt.Fprintln(os.Stderr, "Debug bundle saved to "+utils.Bold(debugDir)) + if len(strings.TrimSpace(sourceCatalog)) > 0 { + fmt.Fprintln(os.Stderr, formatCatalogSummary("Shadow", diff.SummarizeCatalogJSON(sourceCatalog))+ + fmt.Sprintf(" (%s)", formatByteSize(len(sourceCatalog)))) + } + if len(strings.TrimSpace(targetCatalog)) > 0 { + fmt.Fprintln(os.Stderr, formatCatalogSummary("Remote", diff.SummarizeCatalogJSON(targetCatalog))+ + fmt.Sprintf(" (%s)", formatByteSize(len(targetCatalog)))) + } else { + fmt.Fprintln(os.Stderr, "Remote catalog: export failed or empty (inspect connection.txt and pgdelta-stderr.txt)") + } +} + +func formatConnectionInfo(config pgconn.Config) string { + return fmt.Sprintf( + "host=%s port=%d user=%s database=%s url=%s", + config.Host, + config.Port, + config.User, + config.Database, + redactPostgresURL(utils.ToPostgresURL(config)), + ) +} + +func redactPostgresURL(raw string) string { + parsed, err := url.Parse(raw) + if err != nil { + return "" + } + if parsed.User != nil { + username := parsed.User.Username() + if username == "" { + parsed.User = url.UserPassword("redacted", "xxxxx") + } else { + parsed.User = url.UserPassword(username, "xxxxx") + } + } + return parsed.String() +} + +func formatCatalogSummary(label string, summary diff.CatalogSummary) string { + if summary.TotalObjects == 0 { + return label + " catalog: no objects detected" + } + parts := make([]string, 0, len(summary.BySchema)) + for schema, count := range summary.BySchema { + parts = append(parts, fmt.Sprintf("%s=%d", schema, count)) + } + return fmt.Sprintf("%s catalog: %d objects (%s)", label, summary.TotalObjects, strings.Join(parts, ", ")) +} + +func formatByteSize(size int) string { + switch { + case size >= 1<<20: + return fmt.Sprintf("%.1f MB", float64(size)/(1<<20)) + case size >= 1<<10: + return fmt.Sprintf("%.1f KB", float64(size)/(1<<10)) + default: + return fmt.Sprintf("%d B", size) + } +} diff --git a/apps/cli-go/internal/db/pull/pgdelta_pull_debug_test.go b/apps/cli-go/internal/db/pull/pgdelta_pull_debug_test.go new file mode 100644 index 0000000000..6a25edb1a4 --- /dev/null +++ b/apps/cli-go/internal/db/pull/pgdelta_pull_debug_test.go @@ -0,0 +1,96 @@ +package pull + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/jackc/pgconn" + "github.com/jackc/pgx/v4" + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/supabase/cli/internal/db/diff" + "github.com/supabase/cli/internal/utils" +) + +func TestSaveEmptyPgDeltaPullDebug(t *testing.T) { + t.Setenv("PGDELTA_DEBUG", "1") + fsys := afero.NewMemMapFs() + original := exportCatalogPgDelta + t.Cleanup(func() { + exportCatalogPgDelta = original + }) + exportCatalogPgDelta = func(ctx context.Context, targetRef, role string, options ...func(*pgx.ConnConfig)) (string, error) { + return `{"schema":"public","name":"airports"}`, nil + } + config := pgconn.Config{ + Host: "db.example.supabase.co", + Port: 5432, + User: "postgres", + Password: "secret", + Database: "postgres", + } + capture := &diff.PgDeltaDebugCapture{ + SourceCatalog: `{"schema":"public","name":"roles"}`, + Stderr: `{"statementCount":0}`, + } + debugDir, err := saveEmptyPgDeltaPullDebug(context.Background(), config, capture, fsys) + require.NoError(t, err) + require.NotEmpty(t, debugDir) + + sourcePath := filepath.Join(debugDir, "source-catalog.json") + targetPath := filepath.Join(debugDir, "target-catalog.json") + stderrPath := filepath.Join(debugDir, "pgdelta-stderr.txt") + connectionPath := filepath.Join(debugDir, "connection.txt") + errorPath := filepath.Join(debugDir, "error.txt") + + source, err := afero.ReadFile(fsys, sourcePath) + require.NoError(t, err) + assert.Contains(t, string(source), `"roles"`) + + target, err := afero.ReadFile(fsys, targetPath) + require.NoError(t, err) + assert.Contains(t, string(target), `"airports"`) + + stderr, err := afero.ReadFile(fsys, stderrPath) + require.NoError(t, err) + assert.Contains(t, string(stderr), `"statementCount":0`) + + connection, err := afero.ReadFile(fsys, connectionPath) + require.NoError(t, err) + assert.Contains(t, string(connection), "db.example.supabase.co") + assert.NotContains(t, string(connection), "secret") + + errorText, err := afero.ReadFile(fsys, errorPath) + require.NoError(t, err) + assert.Contains(t, string(errorText), "No schema changes found") +} + +func TestSaveEmptyPgDeltaPullDebugUsesTempDir(t *testing.T) { + fsys := afero.NewMemMapFs() + original := exportCatalogPgDelta + t.Cleanup(func() { + exportCatalogPgDelta = original + }) + exportCatalogPgDelta = func(ctx context.Context, targetRef, role string, options ...func(*pgx.ConnConfig)) (string, error) { + return `{}`, nil + } + debugDir, err := saveEmptyPgDeltaPullDebug(context.Background(), pgconn.Config{}, &diff.PgDeltaDebugCapture{}, fsys) + require.NoError(t, err) + assert.Contains(t, debugDir, filepath.Join(utils.TempDir, "pgdelta", "debug")) +} + +func TestDiffRemoteSchemaEmptyWithoutDebug(t *testing.T) { + t.Setenv("PGDELTA_DEBUG", "") + fsys := afero.NewMemMapFs() + existsBefore, err := afero.Exists(fsys, filepath.Join(utils.TempDir, "pgdelta")) + require.NoError(t, err) + assert.False(t, existsBefore) + + // saveEmptyPgDeltaPullDebug should not run when env is unset; verify gate directly. + assert.False(t, diff.IsPgDeltaDebugEnabled()) + _, err = os.Stat(filepath.Join(utils.TempDir, "pgdelta", "debug")) + assert.Error(t, err) +} diff --git a/apps/cli-go/internal/db/pull/pull.go b/apps/cli-go/internal/db/pull/pull.go index 758cb74810..e536c0e990 100644 --- a/apps/cli-go/internal/db/pull/pull.go +++ b/apps/cli-go/internal/db/pull/pull.go @@ -60,7 +60,10 @@ func Run(ctx context.Context, schema []string, config pgconn.Config, name string // 2. Pull schema timestamp := utils.GetCurrentTimestamp() path := new.GetMigrationPath(timestamp, name) - if err := run(ctx, schema, path, conn, usePgDeltaDiff, differ, fsys); err != nil { + if err := run(ctx, schema, path, conn, usePgDeltaDiff, differ, fsys, options...); err != nil { + return err + } + if err := ensureMigrationWritten(fsys, path); err != nil { return err } // 3. Insert a row to `schema_migrations` @@ -110,7 +113,7 @@ func pullDeclarativePgDelta(ctx context.Context, schema []string, config pgconn. return nil } -func run(ctx context.Context, schema []string, path string, conn *pgx.Conn, usePgDeltaDiff bool, differ diff.DiffFunc, fsys afero.Fs) error { +func run(ctx context.Context, schema []string, path string, conn *pgx.Conn, usePgDeltaDiff bool, differ diff.DiffFunc, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { config := conn.Config().Config // 1. Assert `supabase/migrations` and `schema_migrations` are in sync. if err := assertRemoteInSync(ctx, conn, fsys); errors.Is(err, errMissing) { @@ -128,15 +131,13 @@ func run(ctx context.Context, schema []string, path string, conn *pgx.Conn, useP // For the legacy path this is a second pass that captures changes // pg_dump cannot emit (default privileges, managed schemas). For the // pg-delta path this is the only pass and produces the full schema. - if err = diffRemoteSchema(ctx, nil, path, config, usePgDeltaDiff, differ, fsys); errors.Is(err, errInSync) { - err = nil - } + err = swallowInitialInSync(diffRemoteSchema(ctx, nil, path, config, usePgDeltaDiff, differ, fsys, options...), fsys, path) return err } else if err != nil { return err } // 2. Fetch remote schema changes - return diffRemoteSchema(ctx, schema, path, config, usePgDeltaDiff, differ, fsys) + return diffRemoteSchema(ctx, schema, path, config, usePgDeltaDiff, differ, fsys, options...) } func dumpRemoteSchema(ctx context.Context, path string, config pgconn.Config, fsys afero.Fs) error { @@ -153,13 +154,21 @@ func dumpRemoteSchema(ctx context.Context, path string, config pgconn.Config, fs return migration.DumpSchema(ctx, config, f, dump.DockerExec) } -func diffRemoteSchema(ctx context.Context, schema []string, path string, config pgconn.Config, usePgDeltaDiff bool, differ diff.DiffFunc, fsys afero.Fs) error { +func diffRemoteSchema(ctx context.Context, schema []string, path string, config pgconn.Config, usePgDeltaDiff bool, differ diff.DiffFunc, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { // Diff remote db (source) & shadow db (target) and write it as a new migration. - output, err := diff.DiffDatabase(ctx, schema, config, os.Stderr, fsys, differ, usePgDeltaDiff) + result, err := diff.DiffDatabase(ctx, schema, config, os.Stderr, fsys, differ, usePgDeltaDiff, options...) if err != nil { return err } + output := result.SQL if trimmed := strings.TrimSpace(output); len(trimmed) == 0 { + if usePgDeltaDiff && diff.IsPgDeltaDebugEnabled() { + if debugDir, debugErr := saveEmptyPgDeltaPullDebug(ctx, config, result.Debug, fsys, options...); debugErr != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to save pg-delta debug bundle: %v\n", debugErr) + } else if len(debugDir) > 0 { + return errors.Errorf("%w (debug bundle: %s)", errInSync, debugDir) + } + } return errors.New(errInSync) } if err := utils.MkdirIfNotExistFS(fsys, filepath.Dir(path)); err != nil { @@ -227,6 +236,25 @@ func assertRemoteInSync(ctx context.Context, conn *pgx.Conn, fsys afero.Fs) erro return nil } +func hasMigrationContent(fsys afero.Fs, path string) bool { + info, err := fsys.Stat(path) + return err == nil && info.Size() > 0 +} + +func swallowInitialInSync(err error, fsys afero.Fs, path string) error { + if errors.Is(err, errInSync) && hasMigrationContent(fsys, path) { + return nil + } + return err +} + +func ensureMigrationWritten(fsys afero.Fs, path string) error { + if hasMigrationContent(fsys, path) { + return nil + } + return errors.New(errInSync) +} + func suggestMigrationRepair(extraRemote, extraLocal []string) string { result := fmt.Sprintln("\nMake sure your local git repo is up-to-date. If the error persists, try repairing the migration history table:") for _, version := range extraRemote { diff --git a/apps/cli-go/internal/db/pull/pull_test.go b/apps/cli-go/internal/db/pull/pull_test.go index 19a1161850..e9fbae9198 100644 --- a/apps/cli-go/internal/db/pull/pull_test.go +++ b/apps/cli-go/internal/db/pull/pull_test.go @@ -136,6 +136,43 @@ func TestPullSchema(t *testing.T) { }) } +func TestInitialPullInSync(t *testing.T) { + fsys := afero.NewMemMapFs() + path := "0_test.sql" + + t.Run("swallows errInSync when pg_dump already wrote migration content", func(t *testing.T) { + require.NoError(t, afero.WriteFile(fsys, path, []byte("create table t(id int);"), 0644)) + err := swallowInitialInSync(errInSync, fsys, path) + assert.NoError(t, err) + }) + + t.Run("returns errInSync for pg-delta initial pull with no migration file", func(t *testing.T) { + err := swallowInitialInSync(errInSync, fsys, "missing.sql") + assert.ErrorIs(t, err, errInSync) + }) + + t.Run("returns errInSync when migration file is empty", func(t *testing.T) { + require.NoError(t, afero.WriteFile(fsys, "empty.sql", []byte{}, 0644)) + err := swallowInitialInSync(errInSync, fsys, "empty.sql") + assert.ErrorIs(t, err, errInSync) + }) +} + +func TestEnsureMigrationWritten(t *testing.T) { + fsys := afero.NewMemMapFs() + + t.Run("passes when migration file has content", func(t *testing.T) { + path := "0_test.sql" + require.NoError(t, afero.WriteFile(fsys, path, []byte("create table t(id int);"), 0644)) + assert.NoError(t, ensureMigrationWritten(fsys, path)) + }) + + t.Run("returns errInSync when migration file is missing", func(t *testing.T) { + err := ensureMigrationWritten(fsys, "missing.sql") + assert.ErrorIs(t, err, errInSync) + }) +} + func TestSyncRemote(t *testing.T) { t.Run("throws error on permission denied", func(t *testing.T) { // Setup in-memory fs diff --git a/apps/cli-go/internal/gen/types/pgdelta_conn.go b/apps/cli-go/internal/gen/types/pgdelta_conn.go new file mode 100644 index 0000000000..994eecc3b9 --- /dev/null +++ b/apps/cli-go/internal/gen/types/pgdelta_conn.go @@ -0,0 +1,114 @@ +package types + +import ( + "context" + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/jackc/pgx/v4" +) + +const ( + PgDeltaSourceSSLRootCert = "PGDELTA_SOURCE_SSLROOTCERT" + PgDeltaTargetSSLRootCert = "PGDELTA_TARGET_SSLROOTCERT" + pgDeltaCABundleRelPath = "supabase/.temp/pgdelta/supabase-ca-bundle.crt" +) + +func isPostgresURL(ref string) bool { + return strings.HasPrefix(ref, "postgres://") || strings.HasPrefix(ref, "postgresql://") +} + +func isSupabaseHostedPostgresURL(dbURL string) bool { + parsed, err := url.Parse(dbURL) + if err != nil { + return false + } + host := strings.ToLower(parsed.Hostname()) + return strings.HasSuffix(host, ".supabase.co") || + strings.Contains(host, "pooler.supabase.com") +} + +// pgDeltaRootCA returns the CA bundle pg-delta should use for a Postgres URL. +// Supabase-hosted databases always receive the embedded bundle even when the +// SSL probe is skipped (for example in --debug mode). +func pgDeltaRootCA(ctx context.Context, dbURL string, options ...func(*pgx.ConnConfig)) (string, error) { + ca, err := GetRootCA(ctx, dbURL, options...) + if err != nil { + return "", err + } + if len(ca) > 0 { + return ca, nil + } + if isSupabaseHostedPostgresURL(dbURL) { + return caStaging + caProd + caSnap, nil + } + return "", nil +} + +// PreparePgDeltaPostgresRef configures a Postgres URL and env vars for pg-delta. +// +// pg-delta disables TLS when sslmode is absent and only reads PGDELTA_*_SSLROOTCERT +// for verify-ca/verify-full. Remote Supabase databases require verify-ca plus a +// CA bundle written into the workspace so edge-runtime can read it from disk. +func PreparePgDeltaPostgresRef( + ctx context.Context, + ref string, + sslRootCertEnv string, + options ...func(*pgx.ConnConfig), +) (string, []string, error) { + if !isPostgresURL(ref) { + return ref, nil, nil + } + ca, err := pgDeltaRootCA(ctx, ref, options...) + if err != nil { + return "", nil, err + } + if len(ca) == 0 { + return ref, nil, nil + } + containerCertPath, err := writePgDeltaCABundleFile(ca) + if err != nil { + return "", nil, err + } + return ensurePgDeltaSSL(ref, containerCertPath), []string{sslRootCertEnv + "=" + ca}, nil +} + +func writePgDeltaCABundleFile(ca string) (string, error) { + cwd, err := os.Getwd() + if err != nil { + return "", err + } + abs := filepath.Join(cwd, pgDeltaCABundleRelPath) + if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil { + return "", err + } + if err := os.WriteFile(abs, []byte(ca), 0o644); err != nil { + return "", err + } + return "/workspace/" + filepath.ToSlash(pgDeltaCABundleRelPath), nil +} + +func ensurePgDeltaSSL(dbURL, sslrootcertPath string) string { + parsed, err := url.Parse(dbURL) + if err != nil { + return dbURL + } + query := parsed.Query() + switch query.Get("sslmode") { + case "verify-ca", "verify-full": + default: + query.Set("sslmode", "verify-ca") + } + if len(sslrootcertPath) > 0 { + query.Set("sslrootcert", sslrootcertPath) + } + parsed.RawQuery = query.Encode() + return parsed.String() +} + +// EnsurePgDeltaVerifyCA is kept for tests that assert URL sslmode behaviour. +func EnsurePgDeltaVerifyCA(dbURL string) string { + return ensurePgDeltaSSL(dbURL, "") +} diff --git a/apps/cli-go/internal/gen/types/pgdelta_conn_test.go b/apps/cli-go/internal/gen/types/pgdelta_conn_test.go new file mode 100644 index 0000000000..eb29a279ef --- /dev/null +++ b/apps/cli-go/internal/gen/types/pgdelta_conn_test.go @@ -0,0 +1,53 @@ +package types + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestEnsurePgDeltaVerifyCA(t *testing.T) { + t.Run("adds verify-ca when sslmode is absent", func(t *testing.T) { + input := "postgresql://postgres:secret@db.example.supabase.co:5432/postgres?connect_timeout=10" + got := EnsurePgDeltaVerifyCA(input) + assert.Contains(t, got, "sslmode=verify-ca") + assert.Contains(t, got, "connect_timeout=10") + }) + + t.Run("preserves existing verify-ca", func(t *testing.T) { + input := "postgresql://postgres:secret@db.example.supabase.co:5432/postgres?sslmode=verify-ca" + assert.Equal(t, input, EnsurePgDeltaVerifyCA(input)) + }) + + t.Run("preserves existing verify-full", func(t *testing.T) { + input := "postgresql://postgres:secret@db.example.supabase.co:5432/postgres?sslmode=verify-full" + assert.Equal(t, input, EnsurePgDeltaVerifyCA(input)) + }) + + t.Run("replaces require with verify-ca", func(t *testing.T) { + input := "postgresql://postgres:secret@db.example.supabase.co:5432/postgres?sslmode=require" + got := EnsurePgDeltaVerifyCA(input) + assert.Contains(t, got, "sslmode=verify-ca") + assert.NotContains(t, got, "sslmode=require") + }) +} + +func TestEnsurePgDeltaSSLAddsRootCertPath(t *testing.T) { + input := "postgresql://postgres:secret@db.example.supabase.co:5432/postgres?connect_timeout=10" + got := ensurePgDeltaSSL(input, "/workspace/supabase/.temp/pgdelta/supabase-ca-bundle.crt") + assert.Contains(t, got, "sslmode=verify-ca") + assert.Contains(t, got, "sslrootcert=%2Fworkspace%2Fsupabase%2F.temp%2Fpgdelta%2Fsupabase-ca-bundle.crt") +} + +func TestIsSupabaseHostedPostgresURL(t *testing.T) { + assert.True(t, isSupabaseHostedPostgresURL("postgresql://postgres@db.ref.supabase.co:5432/postgres")) + assert.True(t, isSupabaseHostedPostgresURL("postgresql://supabase_admin@aws-0-us-east-2.pooler.supabase.com:5432/postgres")) + assert.False(t, isSupabaseHostedPostgresURL("postgresql://postgres@localhost:5432/postgres")) +} + +func TestPreparePgDeltaPostgresRefNonPostgres(t *testing.T) { + ref, env, err := PreparePgDeltaPostgresRef(t.Context(), "supabase/.temp/catalog.json", PgDeltaTargetSSLRootCert) + assert.NoError(t, err) + assert.Equal(t, "supabase/.temp/catalog.json", ref) + assert.Empty(t, env) +} diff --git a/apps/cli/package.json b/apps/cli/package.json index 0f5041fb34..99877f1af5 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -24,7 +24,8 @@ "access": "public" }, "scripts": { - "build": "pnpm build:next && pnpm build:legacy && pnpm build:shim", + "build": "pnpm build:go-sidecar && pnpm build:next && pnpm build:legacy && pnpm build:shim", + "build:go-sidecar": "cp ../cli-go/supabase-go dist/supabase-go", "build:next": "bun build src/next/main.ts --compile --outfile dist/supabase-next", "build:legacy": "bun build src/legacy/main.ts --compile --outfile dist/supabase-legacy", "build:shim": "bun build src/shared/cli/bin.ts --outfile dist/supabase.js --target node", From b8635d417933b5f98a46cbcceb712cd763e5bdc2 Mon Sep 17 00:00:00 2001 From: avallete Date: Thu, 21 May 2026 21:22:30 +0200 Subject: [PATCH 3/3] feat(cli): enhance local pg-delta testing support Added documentation for testing local pg-delta builds in CONTRIBUTING.md, detailing the steps to publish local changes and configure the CLI to use a local npm registry. Updated the RunEdgeRuntimeScript function to accept additional options for handling npm registry configurations, allowing for scoped .npmrc files and environment variable forwarding. Introduced PgDeltaNpmRegistryOption to manage npm registry settings and added tests to ensure correct behavior. Refactored related functions to integrate these enhancements, improving the overall local development experience for pg-delta. --- apps/cli-go/CONTRIBUTING.md | 49 ++++++++++++ apps/cli-go/internal/db/diff/diff_test.go | 8 +- apps/cli-go/internal/db/diff/pgdelta.go | 6 +- apps/cli-go/internal/db/pgcache/cache.go | 2 +- apps/cli-go/internal/pgdelta/apply.go | 2 +- apps/cli-go/internal/utils/edgeruntime.go | 78 +++++++++++++++++-- .../cli-go/internal/utils/edgeruntime_test.go | 47 +++++++++++ apps/cli-go/internal/utils/pgdelta_local.go | 44 +++++++++++ .../internal/utils/pgdelta_local_test.go | 63 +++++++++++++++ apps/cli-go/pkg/config/config_test.go | 2 +- apps/cli-go/pkg/config/pgdelta_local.go | 14 ++++ 11 files changed, 299 insertions(+), 16 deletions(-) create mode 100644 apps/cli-go/internal/utils/edgeruntime_test.go create mode 100644 apps/cli-go/internal/utils/pgdelta_local.go create mode 100644 apps/cli-go/internal/utils/pgdelta_local_test.go create mode 100644 apps/cli-go/pkg/config/pgdelta_local.go diff --git a/apps/cli-go/CONTRIBUTING.md b/apps/cli-go/CONTRIBUTING.md index 00473f3139..39d0d33d85 100644 --- a/apps/cli-go/CONTRIBUTING.md +++ b/apps/cli-go/CONTRIBUTING.md @@ -41,3 +41,52 @@ go test ./... -race -v -count=1 -failfast ## API client The Supabase API client is generated from OpenAPI spec. See [our guide](api/README.md) for updating the client and types. + +## Testing local pg-delta builds + +To exercise unpublished `@supabase/pg-delta` changes inside CLI edge-runtime scripts (`db pull`, `db diff`, `db push`, etc.), publish a local build via Verdaccio in [pg-toolbelt](https://github.com/supabase/pg-toolbelt) and point the CLI at that registry. + +### 1. Start Verdaccio (pg-toolbelt) + +```sh +cd pg-toolbelt +bun run verdaccio:start +``` + +Verdaccio listens on `http://localhost:4873`. `@supabase/*` packages you publish locally are served from local storage; other `@supabase/*` dependencies (for example `@supabase/pg-topo`) are proxied to npmjs. + +### 2. Publish a local pg-delta build + +After changing `packages/pg-delta`: + +```sh +bun run pg-delta:publish-local \ + --write-version-to=/path/to/test-project/supabase/.temp/pgdelta-version +``` + +This publishes a fresh `0.0.0-local.` version and restores `package.json` afterward. The version file tells the CLI which npm version to request (`EffectivePgDeltaNpmVersion`). + +Re-run whenever you change pg-delta source. + +### 3. Run the CLI against the local registry + +Set `PGDELTA_NPM_REGISTRY` to a URL reachable **from inside the edge-runtime Docker container**: + +```sh +# Docker Desktop (macOS / Windows) +export PGDELTA_NPM_REGISTRY=http://host.docker.internal:4873 + +# Linux (Docker 20.10+) +export PGDELTA_NPM_REGISTRY=http://host.docker.internal:4873 +# or: export PGDELTA_NPM_REGISTRY=http://172.17.0.1:4873 +``` + +Then run any pg-delta-backed command, for example: + +```sh +supabase db pull --db-url "$DATABASE_URL" --diff-engine pg-delta +``` + +When set, the CLI injects a scoped `.npmrc` and forwards `NPM_CONFIG_REGISTRY` into the edge-runtime container (`PgDeltaNpmRegistryOption` in `internal/utils/pgdelta_local.go`). + +Unset `PGDELTA_NPM_REGISTRY` to return to the npmjs version pinned in config / `supabase/.temp/pgdelta-version`. diff --git a/apps/cli-go/internal/db/diff/diff_test.go b/apps/cli-go/internal/db/diff/diff_test.go index f14c6a9f73..fb49df6ff8 100644 --- a/apps/cli-go/internal/db/diff/diff_test.go +++ b/apps/cli-go/internal/db/diff/diff_test.go @@ -205,7 +205,7 @@ func TestDiffDatabase(t *testing.T) { // Run test result, err := DiffDatabase(context.Background(), []string{"public"}, dbConfig, io.Discard, fsys, DiffSchemaMigra, false) // Check error - assert.Empty(t, result.SQL) + assert.Empty(t, result) assert.ErrorIs(t, err, errNetwork) assert.Empty(t, apitest.ListUnmatchedRequests()) }) @@ -236,7 +236,7 @@ func TestDiffDatabase(t *testing.T) { // Run test result, err := DiffDatabase(context.Background(), []string{"public"}, dbConfig, io.Discard, fsys, DiffSchemaMigra, false) // Check error - assert.Empty(t, result.SQL) + assert.Empty(t, result) assert.ErrorContains(t, err, "test-shadow-db container is not running: exited") assert.Empty(t, apitest.ListUnmatchedRequests()) }) @@ -268,7 +268,7 @@ func TestDiffDatabase(t *testing.T) { // Run test result, err := DiffDatabase(context.Background(), []string{"public"}, dbConfig, io.Discard, fsys, DiffSchemaMigra, false, conn.Intercept) // Check error - assert.Empty(t, result.SQL) + assert.Empty(t, result) assert.ErrorContains(t, err, `ERROR: schema "public" already exists (SQLSTATE 42P06) At statement: 0 create schema public`) @@ -333,7 +333,7 @@ create schema public`) } }) // Check error - assert.Empty(t, result.SQL) + assert.Empty(t, result) assert.ErrorContains(t, err, "error diffing schema") assert.Empty(t, apitest.ListUnmatchedRequests()) }) diff --git a/apps/cli-go/internal/db/diff/pgdelta.go b/apps/cli-go/internal/db/diff/pgdelta.go index 787e56cb33..d4a63e1634 100644 --- a/apps/cli-go/internal/db/diff/pgdelta.go +++ b/apps/cli-go/internal/db/diff/pgdelta.go @@ -133,7 +133,7 @@ func DiffPgDeltaRefDetailed(ctx context.Context, sourceRef, targetRef string, sc } var stdout, stderr bytes.Buffer script := config.InterpolatePgDeltaScript(config.Config(&utils.Config), pgDeltaScript) - if err := utils.RunEdgeRuntimeScript(ctx, env, script, binds, "error diffing schema", &stdout, &stderr); err != nil { + if err := utils.RunEdgeRuntimeScript(ctx, env, script, binds, "error diffing schema", &stdout, &stderr, utils.PgDeltaNpmRegistryOption()); err != nil { return PgDeltaDiffResult{}, err } return PgDeltaDiffResult{ @@ -181,7 +181,7 @@ func DeclarativeExportPgDeltaRef(ctx context.Context, sourceRef, targetRef strin } var stdout, stderr bytes.Buffer script := config.InterpolatePgDeltaScript(config.Config(&utils.Config), pgDeltaDeclarativeExportScript) - if err := utils.RunEdgeRuntimeScript(ctx, env, script, binds, "error exporting declarative schema", &stdout, &stderr); err != nil { + if err := utils.RunEdgeRuntimeScript(ctx, env, script, binds, "error exporting declarative schema", &stdout, &stderr, utils.PgDeltaNpmRegistryOption()); err != nil { return DeclarativeOutput{}, err } if stdout.Len() == 0 { @@ -214,7 +214,7 @@ func ExportCatalogPgDelta(ctx context.Context, targetRef, role string, options . } var stdout, stderr bytes.Buffer script := config.InterpolatePgDeltaScript(config.Config(&utils.Config), pgDeltaCatalogExportScript) - if err := utils.RunEdgeRuntimeScript(ctx, env, script, binds, "error exporting pg-delta catalog", &stdout, &stderr); err != nil { + if err := utils.RunEdgeRuntimeScript(ctx, env, script, binds, "error exporting pg-delta catalog", &stdout, &stderr, utils.PgDeltaNpmRegistryOption()); err != nil { return "", err } snapshot := strings.TrimSpace(stdout.String()) diff --git a/apps/cli-go/internal/db/pgcache/cache.go b/apps/cli-go/internal/db/pgcache/cache.go index ea9bb202aa..5b518464ba 100644 --- a/apps/cli-go/internal/db/pgcache/cache.go +++ b/apps/cli-go/internal/db/pgcache/cache.go @@ -254,7 +254,7 @@ func exportCatalog(ctx context.Context, targetRef string, options ...func(*pgx.C binds := []string{utils.EdgeRuntimeId + ":/root/.cache/deno:rw"} var stdout, stderr bytes.Buffer script := config.InterpolatePgDeltaScript(config.Config(&utils.Config), pgDeltaCatalogExportTS) - if err := utils.RunEdgeRuntimeScript(ctx, env, script, binds, "error exporting pg-delta catalog", &stdout, &stderr); err != nil { + if err := utils.RunEdgeRuntimeScript(ctx, env, script, binds, "error exporting pg-delta catalog", &stdout, &stderr, utils.PgDeltaNpmRegistryOption()); err != nil { return "", err } snapshot := strings.TrimSpace(stdout.String()) diff --git a/apps/cli-go/internal/pgdelta/apply.go b/apps/cli-go/internal/pgdelta/apply.go index f9009a3202..22fca756a6 100644 --- a/apps/cli-go/internal/pgdelta/apply.go +++ b/apps/cli-go/internal/pgdelta/apply.go @@ -323,7 +323,7 @@ func ApplyDeclarative(ctx context.Context, config pgconn.Config, fsys afero.Fs) fmt.Fprintln(os.Stderr, "Applying declarative schemas via pg-delta...") var stdout, stderr bytes.Buffer script := pkgconfig.InterpolatePgDeltaScript(pkgconfig.Config(&utils.Config), pgDeltaDeclarativeApplyScript) - if err := utils.RunEdgeRuntimeScript(ctx, env, script, binds, "error running pg-delta script", &stdout, &stderr); err != nil { + if err := utils.RunEdgeRuntimeScript(ctx, env, script, binds, "error running pg-delta script", &stdout, &stderr, utils.PgDeltaNpmRegistryOption()); err != nil { return err } diff --git a/apps/cli-go/internal/utils/edgeruntime.go b/apps/cli-go/internal/utils/edgeruntime.go index 06a42b464c..7708a15511 100644 --- a/apps/cli-go/internal/utils/edgeruntime.go +++ b/apps/cli-go/internal/utils/edgeruntime.go @@ -3,6 +3,7 @@ package utils import ( "bytes" "context" + "fmt" "strings" "github.com/docker/docker/api/types/container" @@ -11,23 +12,67 @@ import ( "github.com/spf13/viper" ) +// edgeRuntimeFile is a single file dropped into the edge-runtime container's +// working directory before the configured command is run. +type edgeRuntimeFile struct { + name string + content string +} + +// edgeRuntimeOptions accumulates the optional inputs assembled by +// EdgeRuntimeOption functions and consumed by RunEdgeRuntimeScript. +type edgeRuntimeOptions struct { + extraFiles []edgeRuntimeFile + extraEnv []string +} + +// EdgeRuntimeOption customizes a RunEdgeRuntimeScript invocation. The current +// shape (extra files dropped alongside index.ts, extra container env vars) +// covers the local-pg-delta use case; extend the option struct as new needs +// arrive instead of adding more positional arguments. +type EdgeRuntimeOption func(*edgeRuntimeOptions) + +// WithExtraFile schedules an extra file alongside `index.ts` in the container. +// Useful for project-local config files (e.g. `.npmrc`, `deno.json`) that need +// to live next to the script Deno is asked to run. +func WithExtraFile(name, content string) EdgeRuntimeOption { + return func(o *edgeRuntimeOptions) { + o.extraFiles = append(o.extraFiles, edgeRuntimeFile{name: name, content: content}) + } +} + +// WithExtraEnv appends container env entries in `KEY=value` form. +func WithExtraEnv(entries ...string) EdgeRuntimeOption { + return func(o *edgeRuntimeOptions) { + o.extraEnv = append(o.extraEnv, entries...) + } +} + // RunEdgeRuntimeScript executes a TypeScript program inside the configured Edge // Runtime container and streams stdout/stderr back to the caller. -func RunEdgeRuntimeScript(ctx context.Context, env []string, script string, binds []string, errPrefix string, stdout, stderr *bytes.Buffer) error { +func RunEdgeRuntimeScript(ctx context.Context, env []string, script string, binds []string, errPrefix string, stdout, stderr *bytes.Buffer, opts ...EdgeRuntimeOption) error { + state := &edgeRuntimeOptions{} + for _, opt := range opts { + if opt != nil { + opt(state) + } + } cmd := []string{"edge-runtime", "start", "--main-service=."} if viper.GetBool("DEBUG") { cmd = append(cmd, "--verbose") } cmdString := strings.Join(cmd, " ") - entrypoint := []string{"sh", "-c", `cat <<'EOF' > index.ts && ` + cmdString + ` -` + script + ` -EOF -`} + files := append([]edgeRuntimeFile{{name: "index.ts", content: script}}, state.extraFiles...) + entrypoint := []string{"sh", "-c", buildEdgeRuntimeEntrypoint(files, cmdString)} + combinedEnv := env + if len(state.extraEnv) > 0 { + combinedEnv = append(append([]string{}, env...), state.extraEnv...) + } if err := DockerRunOnceWithConfig( ctx, container.Config{ Image: Config.EdgeRuntime.Image, - Env: env, + Env: combinedEnv, Entrypoint: entrypoint, }, container.HostConfig{ @@ -43,3 +88,24 @@ EOF } return nil } + +// buildEdgeRuntimeEntrypoint emits a `sh -c` body that writes each file via a +// here-document and then runs cmd. All heredoc openers are joined with `&&` +// before the bodies so bash stacks them in declaration order; each body is +// terminated with a unique sentinel so file contents can contain `EOF` safely. +func buildEdgeRuntimeEntrypoint(files []edgeRuntimeFile, cmd string) string { + if len(files) == 0 { + return cmd + "\n" + } + var head strings.Builder + var bodies strings.Builder + for i, f := range files { + sentinel := fmt.Sprintf("__EDGE_RT_FILE_%d__", i) + fmt.Fprintf(&head, "cat <<'%s' > %s && ", sentinel, f.name) + fmt.Fprintf(&bodies, "%s\n%s\n", f.content, sentinel) + } + head.WriteString(cmd) + head.WriteString("\n") + head.WriteString(bodies.String()) + return head.String() +} diff --git a/apps/cli-go/internal/utils/edgeruntime_test.go b/apps/cli-go/internal/utils/edgeruntime_test.go new file mode 100644 index 0000000000..3f2a74ada4 --- /dev/null +++ b/apps/cli-go/internal/utils/edgeruntime_test.go @@ -0,0 +1,47 @@ +package utils + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBuildEdgeRuntimeEntrypoint(t *testing.T) { + t.Run("emits a single heredoc when only the script is provided", func(t *testing.T) { + got := buildEdgeRuntimeEntrypoint( + []edgeRuntimeFile{{name: "index.ts", content: "console.log('hi')"}}, + "edge-runtime start --main-service=.", + ) + assert.True(t, strings.HasPrefix(got, "cat <<'__EDGE_RT_FILE_0__' > index.ts && edge-runtime start --main-service=.\n")) + assert.Contains(t, got, "console.log('hi')\n__EDGE_RT_FILE_0__\n") + }) + + t.Run("chains heredocs in declaration order so each cat reads the matching body", func(t *testing.T) { + got := buildEdgeRuntimeEntrypoint( + []edgeRuntimeFile{ + {name: "index.ts", content: "TS_CONTENT"}, + {name: ".npmrc", content: "NPMRC_CONTENT"}, + }, + "edge-runtime start --main-service=.", + ) + // Both cat declarations must come before any body, separated by &&. + assert.Contains(t, got, "cat <<'__EDGE_RT_FILE_0__' > index.ts && cat <<'__EDGE_RT_FILE_1__' > .npmrc && edge-runtime start --main-service=.") + // Bodies must follow in the same order as the declarations. + idxScript := strings.Index(got, "TS_CONTENT") + idxNpmrc := strings.Index(got, "NPMRC_CONTENT") + require.Greater(t, idxScript, 0) + require.Greater(t, idxNpmrc, idxScript, ".npmrc body must come after index.ts body") + // Sentinels close each body so user content containing `EOF` cannot + // terminate the heredoc early. + assert.Contains(t, got, "TS_CONTENT\n__EDGE_RT_FILE_0__") + assert.Contains(t, got, "NPMRC_CONTENT\n__EDGE_RT_FILE_1__") + assert.True(t, strings.HasSuffix(got, "\n")) + }) + + t.Run("returns just the command when no files are provided", func(t *testing.T) { + got := buildEdgeRuntimeEntrypoint(nil, "edge-runtime start --main-service=.") + assert.Equal(t, "edge-runtime start --main-service=.\n", got) + }) +} diff --git a/apps/cli-go/internal/utils/pgdelta_local.go b/apps/cli-go/internal/utils/pgdelta_local.go new file mode 100644 index 0000000000..50232b238e --- /dev/null +++ b/apps/cli-go/internal/utils/pgdelta_local.go @@ -0,0 +1,44 @@ +package utils + +import ( + "os" + "strings" + + "github.com/supabase/cli/pkg/config" +) + +// PgDeltaNpmRegistryOption returns an EdgeRuntimeOption that points the +// edge-runtime container at a user-controlled npm registry when +// PGDELTA_NPM_REGISTRY is set. It applies three coordinated overrides: +// +// 1. Writes a project-local `.npmrc` with a `@supabase`-scoped registry +// line. Deno honors `.npmrc` for scoped registries when discovered in +// the cwd or parents (Deno >= 1.39), so this keeps every non-`@supabase` +// npm specifier on npmjs. +// 2. Forwards the canonical `NPM_CONFIG_REGISTRY` env var into the +// container. This is the universal npm/Deno escape hatch — it routes +// every `npm:` specifier through the chosen registry regardless of +// whether the host runtime reads `.npmrc`. Verdaccio's `npmjs` uplink +// proxies any non-`@supabase` packages back to npmjs, so widening the +// scope is safe and protects us against edge-runtime image variants +// that ignore `.npmrc`. +// 3. Forwards `PGDELTA_NPM_REGISTRY` itself into the container. +// +// Returns nil when the env var is unset or whitespace-only, which makes it +// safe to pass unconditionally to RunEdgeRuntimeScript (nil options are +// ignored). +func PgDeltaNpmRegistryOption() EdgeRuntimeOption { + registry := strings.TrimSpace(os.Getenv(config.PgDeltaNpmRegistryEnv)) + if registry == "" { + return nil + } + npmrc := WithExtraFile(".npmrc", "@supabase:registry="+registry+"\n") + envFwd := WithExtraEnv( + config.PgDeltaNpmRegistryEnv+"="+registry, + "NPM_CONFIG_REGISTRY="+registry, + ) + return func(o *edgeRuntimeOptions) { + npmrc(o) + envFwd(o) + } +} diff --git a/apps/cli-go/internal/utils/pgdelta_local_test.go b/apps/cli-go/internal/utils/pgdelta_local_test.go new file mode 100644 index 0000000000..656e17c4b2 --- /dev/null +++ b/apps/cli-go/internal/utils/pgdelta_local_test.go @@ -0,0 +1,63 @@ +package utils + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/supabase/cli/pkg/config" +) + +func TestPgDeltaNpmRegistryOption(t *testing.T) { + t.Run("returns nil when PGDELTA_NPM_REGISTRY is unset", func(t *testing.T) { + t.Setenv(config.PgDeltaNpmRegistryEnv, "") + assert.Nil(t, PgDeltaNpmRegistryOption()) + }) + + t.Run("writes a scoped .npmrc and forwards both PGDELTA_NPM_REGISTRY and NPM_CONFIG_REGISTRY when set", func(t *testing.T) { + t.Setenv(config.PgDeltaNpmRegistryEnv, "http://host.docker.internal:4873") + opt := PgDeltaNpmRegistryOption() + require.NotNil(t, opt) + + state := &edgeRuntimeOptions{} + opt(state) + require.Len(t, state.extraFiles, 1) + assert.Equal(t, ".npmrc", state.extraFiles[0].name) + assert.Equal(t, + "@supabase:registry=http://host.docker.internal:4873\n", + state.extraFiles[0].content, + ) + // NPM_CONFIG_REGISTRY is the universal escape hatch for runtimes + // that ignore .npmrc (e.g. some supabase/edge-runtime variants); + // PGDELTA_NPM_REGISTRY is forwarded so scripts can read the configured + // registry URL when needed. + assert.Equal(t, + []string{ + "PGDELTA_NPM_REGISTRY=http://host.docker.internal:4873", + "NPM_CONFIG_REGISTRY=http://host.docker.internal:4873", + }, + state.extraEnv, + ) + }) + + t.Run("trims surrounding whitespace from the registry URL", func(t *testing.T) { + t.Setenv(config.PgDeltaNpmRegistryEnv, " http://localhost:4873 ") + opt := PgDeltaNpmRegistryOption() + require.NotNil(t, opt) + + state := &edgeRuntimeOptions{} + opt(state) + require.Len(t, state.extraFiles, 1) + assert.Equal(t, + "@supabase:registry=http://localhost:4873\n", + state.extraFiles[0].content, + ) + assert.Equal(t, + []string{ + "PGDELTA_NPM_REGISTRY=http://localhost:4873", + "NPM_CONFIG_REGISTRY=http://localhost:4873", + }, + state.extraEnv, + ) + }) +} diff --git a/apps/cli-go/pkg/config/config_test.go b/apps/cli-go/pkg/config/config_test.go index d7bca3948d..06dd1e59a7 100644 --- a/apps/cli-go/pkg/config/config_test.go +++ b/apps/cli-go/pkg/config/config_test.go @@ -288,7 +288,7 @@ enabled = true t.Run("InterpolatePgDeltaScript substitutes placeholder", func(t *testing.T) { c := NewConfig() require.NoError(t, c.Load("", fs.MapFS{})) - // Embedded TS pins use this semver literal before InterpolatePgDeltaScript runs. + // Embedded TS pins use this semver literal before InterpolatePgDeltaScript runs. got := InterpolatePgDeltaScript(Config(&c), `from "npm:@supabase/pg-delta@1.0.0-alpha.20";`) assert.Equal(t, `from "npm:@supabase/pg-delta@`+DefaultPgDeltaNpmVersion+`";`, got) }) diff --git a/apps/cli-go/pkg/config/pgdelta_local.go b/apps/cli-go/pkg/config/pgdelta_local.go new file mode 100644 index 0000000000..683137f1b0 --- /dev/null +++ b/apps/cli-go/pkg/config/pgdelta_local.go @@ -0,0 +1,14 @@ +package config + +// PgDeltaNpmRegistryEnv is the env var that, when set to an npm registry URL +// reachable from the edge-runtime container, routes Deno's `npm:` resolution +// for `@supabase/pg-delta` through that registry instead of the public +// npmjs.org. Pair with the pg-toolbelt `bun run pg-delta:publish-local` script +// to iterate on local pg-delta changes without republishing to npmjs. +// +// See apps/cli-go/CONTRIBUTING.md#testing-local-pg-delta-builds for the +// Verdaccio workflow (CLI maintainers only). +// +// Typical value when running pg-toolbelt's Verdaccio on Docker Desktop: +// PGDELTA_NPM_REGISTRY=http://host.docker.internal:4873 +const PgDeltaNpmRegistryEnv = "PGDELTA_NPM_REGISTRY"