diff --git a/README.md b/README.md index 5d9fa40..d1d9054 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,7 @@ development: adapter: postgresql_branched database: myapp_development primary_branch: main # default, can be 'master', 'trunk', etc. + unlogged_branches: false # default; set to true to shadow tables UNLOGGED ``` The adapter needs a branch name. Resolution order: @@ -151,6 +152,15 @@ All standard PostgreSQL connection parameters work as normal (`host`, `port`, `u **Do not set `schema_search_path` in database.yml** — it conflicts with the adapter's `search_path` management. +### Unlogged branch shadows + +Set `unlogged_branches: true` to create shadow tables as Postgres `UNLOGGED`. WAL is skipped for shadow writes, which reduces disk pressure on large prod-restored dev databases and shortens shadow time on slow storage. + +Branch schemas are disposable by design — `db:branch:reset` rebuilds them from `public` — so the crash-loss semantics of UNLOGGED are a good match for dev/test. Two caveats: + +- A foreign key added on a feature branch (`add_foreign_key :shadowed, :public_only_table`) fails when the shadow is UNLOGGED and the referenced table is still LOGGED in `public`. Either shadow the reference first, or turn the option off. +- If Postgres crashes, all shadow tables are truncated and you re-run `db:branch:reset` + `db:migrate`. No worse than switching branches. + ## Limitations - **Rails + PostgreSQL only** — uses PostgreSQL schemas and `search_path` diff --git a/lib/active_record/connection_adapters/postgresql/branched/adapter.rb b/lib/active_record/connection_adapters/postgresql/branched/adapter.rb index 4fcb8fc..cbde7d2 100644 --- a/lib/active_record/connection_adapters/postgresql/branched/adapter.rb +++ b/lib/active_record/connection_adapters/postgresql/branched/adapter.rb @@ -31,7 +31,13 @@ class Adapter < ActiveRecord::ConnectionAdapters::PostgreSQLAdapter def initialize(...) super @branch_manager = BranchManager.new(self, @config) - @shadow = Shadow.new(self, @branch_manager.branch_schema) unless @branch_manager.primary_branch? + return if @branch_manager.primary_branch? + + @shadow = Shadow.new( + self, + @branch_manager.branch_schema, + unlogged: @config.fetch(:unlogged_branches, false) + ) end def configure_connection diff --git a/lib/active_record/connection_adapters/postgresql/branched/shadow.rb b/lib/active_record/connection_adapters/postgresql/branched/shadow.rb index 346c577..3cca318 100644 --- a/lib/active_record/connection_adapters/postgresql/branched/shadow.rb +++ b/lib/active_record/connection_adapters/postgresql/branched/shadow.rb @@ -3,9 +3,10 @@ module ConnectionAdapters module PostgreSQL module Branched class Shadow - def initialize(connection, branch_schema) + def initialize(connection, branch_schema, unlogged: false) @connection = connection @branch_schema = branch_schema + @unlogged = unlogged end def call(table_name) @@ -36,15 +37,71 @@ def already_shadowed?(table) def create_shadow(table) quoted_branch = @connection.quote_column_name(@branch_schema) quoted_table = @connection.quote_column_name(table) + + clone_structure(quoted_branch, quoted_table) + copy_data(quoted_branch, quoted_table) + clone_indexes(table, quoted_branch, quoted_table) + end + + # Per-row index maintenance during INSERT SELECT dominates the cost + # of shadowing large tables. LIKE ... INCLUDING ALL builds every + # index up front, then the data copy updates each index for every + # row. EXCLUDING INDEXES lets the bulk copy run without index + # overhead; clone_indexes rebuilds them afterwards in a single + # sort pass per index, the standard Postgres bulk-load pattern. + # + # UNLOGGED (when enabled) skips WAL for both the bulk copy and + # subsequent writes. Branch schemas are disposable by design — + # `db:branch:reset` rebuilds them from public — so the crash-loss + # semantics of UNLOGGED are a good fit for dev/test. + def clone_structure(quoted_branch, quoted_table) @connection.execute(<<~SQL) - CREATE TABLE #{quoted_branch}.#{quoted_table} - (LIKE public.#{quoted_table} INCLUDING ALL) + CREATE #{"UNLOGGED " if @unlogged}TABLE #{quoted_branch}.#{quoted_table} + (LIKE public.#{quoted_table} INCLUDING ALL EXCLUDING INDEXES) SQL + end + + def copy_data(quoted_branch, quoted_table) @connection.execute(<<~SQL) INSERT INTO #{quoted_branch}.#{quoted_table} SELECT * FROM public.#{quoted_table} SQL end + + def clone_indexes(table, quoted_branch, quoted_table) + index_metadata(table).each do |index_name, is_primary, branched_indexdef| + @connection.execute(branched_indexdef) + + next unless is_primary + + quoted_index = @connection.quote_column_name(index_name) + @connection.execute(<<~SQL) + ALTER TABLE #{quoted_branch}.#{quoted_table} + ADD CONSTRAINT #{quoted_index} PRIMARY KEY USING INDEX #{quoted_index} + SQL + end + end + + # Redirect pg_get_indexdef output from public to the branch schema. + # The REPLACE uses the same quote_ident() that pg_get_indexdef uses + # to emit identifiers, so the anchor always matches regardless of + # whether the table name needs quoting. + def index_metadata(table) + @connection.select_rows(<<~SQL) + SELECT i.relname, ix.indisprimary, + REPLACE( + pg_get_indexdef(ix.indexrelid), + ' ON public.' || quote_ident(t.relname), + ' ON ' || quote_ident(#{@connection.quote(@branch_schema)}) || '.' || quote_ident(t.relname) + ) + FROM pg_index ix + JOIN pg_class i ON i.oid = ix.indexrelid + JOIN pg_class t ON t.oid = ix.indrelid + JOIN pg_namespace n ON n.oid = t.relnamespace + WHERE n.nspname = 'public' + AND t.relname = #{@connection.quote(table)} + SQL + end end end end diff --git a/test/shadow_test.rb b/test/shadow_test.rb index f7932b8..20c78f5 100644 --- a/test/shadow_test.rb +++ b/test/shadow_test.rb @@ -244,4 +244,214 @@ def test_shadow_preserves_data_accessible_via_foreign_key conn.execute("INSERT INTO posts (author_id, title, body) VALUES (#{author_id}, 'world', 'content')") assert_equal 2, conn.select_value("SELECT count(*) FROM posts") end + + # --- Bulk-load shadow preserves every structural detail --- + # + # These tests exist because the shadow now skips INCLUDING INDEXES during + # CREATE TABLE LIKE and rebuilds indexes after the data copy. They pin the + # contract that the branch table is structurally equivalent to public. + + def test_shadow_preserves_primary_key_as_primary_key + conn = reconnect(branch: "feature/pk") + conn.add_column :users, :bio, :string + + is_primary = conn.select_value(<<~SQL) + SELECT ix.indisprimary FROM pg_index ix + JOIN pg_class t ON t.oid = ix.indrelid + JOIN pg_namespace n ON n.oid = t.relnamespace + WHERE n.nspname = 'branch_feature_pk' + AND t.relname = 'users' + AND ix.indisprimary + SQL + assert_equal true, is_primary, + "Shadow must attach the pkey index as a PRIMARY KEY constraint, not just a unique index" + end + + def test_shadow_primary_key_sequence_still_works + conn = reconnect(branch: "feature/pk-insert") + conn.add_column :users, :bio, :string + + conn.execute("INSERT INTO users (name, legacy, bio) VALUES ('carol', 'old', 'hi')") + inserted_id = conn.select_value("SELECT id FROM users WHERE name = 'carol'") + refute_nil inserted_id, "Inserts into the shadow must auto-assign an id" + end + + def test_shadow_preserves_unique_index + conn = connect(branch: "main") + conn.execute("CREATE UNIQUE INDEX users_name_uniq ON public.users (name)") + + conn = reconnect(branch: "feature/uniq-idx") + conn.add_column :users, :bio, :string + + assert index_exists_in_schema?(conn, "branch_feature_uniq_idx", "users_name_uniq") + is_unique = conn.select_value(<<~SQL) + SELECT ix.indisunique FROM pg_index ix + JOIN pg_class i ON i.oid = ix.indexrelid + JOIN pg_namespace n ON n.oid = i.relnamespace + WHERE n.nspname = 'branch_feature_uniq_idx' AND i.relname = 'users_name_uniq' + SQL + assert_equal true, is_unique, "Unique index must be rebuilt as unique" + + assert_raises(ActiveRecord::RecordNotUnique) do + conn.execute("INSERT INTO users (name, legacy, bio) VALUES ('alice', 'new', 'dup')") + end + end + + def test_shadow_preserves_non_unique_index + conn = connect(branch: "main") + conn.execute("CREATE INDEX users_legacy_idx ON public.users (legacy)") + + conn = reconnect(branch: "feature/plain-idx") + conn.add_column :users, :bio, :string + + assert index_exists_in_schema?(conn, "branch_feature_plain_idx", "users_legacy_idx") + end + + def test_shadow_preserves_check_constraint + conn = connect(branch: "main") + conn.execute("ALTER TABLE public.orders ADD CONSTRAINT orders_amount_positive CHECK (amount > 0)") + + conn = reconnect(branch: "feature/check") + conn.add_column :orders, :note, :string + + assert check_constraint_exists_in_schema?(conn, "branch_feature_check", "orders", "orders_amount_positive") + assert_raises(ActiveRecord::StatementInvalid) do + conn.execute("INSERT INTO orders (status, amount, note) VALUES ('bad', -1, 'x')") + end + end + + def test_shadow_preserves_not_null_and_default + conn = connect(branch: "main") + conn.execute("ALTER TABLE public.orders ALTER COLUMN status SET NOT NULL") + conn.execute("ALTER TABLE public.orders ALTER COLUMN status SET DEFAULT 'pending'") + + conn = reconnect(branch: "feature/default") + conn.add_column :orders, :note, :string + + is_not_null = conn.select_value(<<~SQL) + SELECT attnotnull FROM pg_attribute a + JOIN pg_class t ON t.oid = a.attrelid + JOIN pg_namespace n ON n.oid = t.relnamespace + WHERE n.nspname = 'branch_feature_default' + AND t.relname = 'orders' + AND a.attname = 'status' + SQL + assert_equal true, is_not_null, "NOT NULL must survive shadow" + + conn.execute("INSERT INTO orders (amount, note) VALUES (50, 'uses-default')") + status = conn.select_value("SELECT status FROM orders WHERE note = 'uses-default'") + assert_equal "pending", status, "Default must survive shadow" + end + + def test_shadow_preserves_multi_column_index + conn = connect(branch: "main") + conn.execute("CREATE INDEX orders_status_amount ON public.orders (status, amount)") + + conn = reconnect(branch: "feature/multi-col") + conn.add_column :orders, :note, :string + + columns = conn.select_values(<<~SQL) + SELECT a.attname FROM pg_index ix + JOIN pg_class i ON i.oid = ix.indexrelid + JOIN pg_namespace n ON n.oid = i.relnamespace + JOIN pg_attribute a ON a.attrelid = ix.indrelid AND a.attnum = ANY(ix.indkey) + WHERE n.nspname = 'branch_feature_multi_col' AND i.relname = 'orders_status_amount' + ORDER BY array_position(ix.indkey, a.attnum) + SQL + assert_equal %w[status amount], columns, "Column order in multi-column index must be preserved" + end + + def test_shadow_preserves_functional_and_partial_indexes + conn = connect(branch: "main") + # Functional and partial indexes give pg_get_indexdef more complex output — + # exactly the case where a naive " ON public." string replacement could go + # wrong. The rewrite must still target only the table qualifier. + conn.execute("CREATE INDEX users_lower_name ON public.users (lower(name))") + conn.execute("CREATE INDEX users_non_legacy ON public.users (name) WHERE legacy IS NULL") + + conn = reconnect(branch: "feature/fn-idx") + conn.add_column :users, :bio, :string + + assert index_exists_in_schema?(conn, "branch_feature_fn_idx", "users_lower_name") + assert index_exists_in_schema?(conn, "branch_feature_fn_idx", "users_non_legacy") + end + + def test_shadow_preserves_indexes_on_quoted_table_name + # A table whose name happens to need SQL quoting (capital letters, reserved + # word, etc.) surfaces quoting bugs in the index-rewrite path. pg_get_indexdef + # quotes only when necessary; the adapter's Ruby-side quoting always does. + # Mismatched expectations here made the first take on this rewrite fail. + conn = connect(branch: "main") + conn.execute('CREATE TABLE public."User" (id serial PRIMARY KEY, email varchar)') + conn.execute('CREATE INDEX user_email_idx ON public."User" (email)') + conn.execute("INSERT INTO public.\"User\" (email) VALUES ('a@x.com')") + + conn = reconnect(branch: "feature/quoted-name") + conn.add_column "User", :bio, :string + + assert table_exists_in_schema?(conn, "branch_feature_quoted_name", "User") + assert index_exists_in_schema?(conn, "branch_feature_quoted_name", "user_email_idx") + assert_equal 1, conn.select_value('SELECT count(*) FROM "User"') + end + + # --- unlogged_branches option --- + # + # pg_class.relpersistence values: + # 'p' = permanent (default), 'u' = unlogged, 't' = temporary + + def test_shadow_is_logged_by_default + conn = reconnect(branch: "feature/logged") + conn.add_column :users, :bio, :string + + assert_equal "p", relpersistence(conn, "branch_feature_logged", "users") + end + + def test_shadow_is_unlogged_when_enabled + conn = reconnect(branch: "feature/unlogged", unlogged_branches: true) + conn.add_column :users, :bio, :string + + assert_equal "u", relpersistence(conn, "branch_feature_unlogged", "users") + end + + def test_unlogged_shadow_still_has_primary_key_and_data + conn = reconnect(branch: "feature/unlogged-pk", unlogged_branches: true) + conn.add_column :users, :bio, :string + + assert_equal 2, conn.select_value("SELECT count(*) FROM users"), + "Unlogged shadow must still copy data" + is_primary = conn.select_value(<<~SQL) + SELECT ix.indisprimary FROM pg_index ix + JOIN pg_class t ON t.oid = ix.indrelid + JOIN pg_namespace n ON n.oid = t.relnamespace + WHERE n.nspname = 'branch_feature_unlogged_pk' + AND t.relname = 'users' + AND ix.indisprimary + SQL + assert_equal true, is_primary, "Unlogged shadow must keep the PK designation" + end + + def test_unlogged_indexes_inherit_unlogged_status + conn = connect(branch: "main") + conn.execute("CREATE INDEX users_name_idx ON public.users (name)") + + conn = reconnect(branch: "feature/unlogged-idx", unlogged_branches: true) + conn.add_column :users, :bio, :string + + index_persistence = conn.select_value(<<~SQL) + SELECT c.relpersistence FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'branch_feature_unlogged_idx' + AND c.relname = 'users_name_idx' + SQL + assert_equal "u", index_persistence, + "Indexes on unlogged tables must themselves be unlogged" + end + + def test_unlogged_option_is_ignored_on_primary_branch + conn = reconnect(branch: "main", unlogged_branches: true) + conn.add_column :users, :bio, :string + + assert_equal "p", relpersistence(conn, "public", "users"), + "Primary branch must never create UNLOGGED tables — public holds the canonical data" + end end diff --git a/test/test_helper.rb b/test/test_helper.rb index a3d7c23..c971b81 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -24,19 +24,30 @@ def teardown_test_database ENV.delete("PGBRANCH") end - def connect(branch:, primary_branch: "main") + def connect(branch:, primary_branch: "main", unlogged_branches: nil) ENV["PGBRANCH"] = branch - ActiveRecord::Base.establish_connection( + config = { adapter: "postgresql_branched", database: DATABASE_NAME, primary_branch: primary_branch - ) + } + config[:unlogged_branches] = unlogged_branches unless unlogged_branches.nil? + ActiveRecord::Base.establish_connection(config) ActiveRecord::Base.lease_connection end - def reconnect(branch:, primary_branch: "main") + def reconnect(branch:, primary_branch: "main", unlogged_branches: nil) ActiveRecord::Base.connection_pool.disconnect! - connect(branch: branch, primary_branch: primary_branch) + connect(branch: branch, primary_branch: primary_branch, unlogged_branches: unlogged_branches) + end + + def relpersistence(connection, schema_name, table_name) + connection.select_value(<<~SQL) + SELECT relpersistence FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = #{connection.quote(schema_name)} + AND c.relname = #{connection.quote(table_name)} + SQL end def schema_exists?(connection, schema_name)