From 26d037f8ab70bf1eb2a064d58c8d8b29db5817f3 Mon Sep 17 00:00:00 2001 From: mhenrixon Date: Fri, 17 Apr 2026 17:09:39 +0200 Subject: [PATCH 1/2] Shadow via bulk load: skip indexes during copy, rebuild after MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CREATE TABLE LIKE ... INCLUDING ALL creates all indexes up front, so the subsequent INSERT SELECT pays for per-row index maintenance on every index. On large tables this dominates shadow time. Instead, create the shadow with INCLUDING ALL EXCLUDING INDEXES, bulk copy the rows, then recreate indexes in a single sort pass each. The primary key is attached via ALTER TABLE ... USING INDEX so it remains a PRIMARY KEY constraint rather than just a unique index. Benchmarks on a widgets table with 1 PK + 5 indexes (1 unique, 1 composite, 1 partial, 2 plain): 1M rows: 14.05s -> 5.98s (2.35x) 3M rows: 51.18s -> 19.25s (2.66x) Speedup grows with scale because sort-based index builds amortize better than per-row B-tree inserts. Semantics are preserved: same indexes, same PK designation, same uniqueness, same check constraints, same defaults and NOT NULL — covered by new tests pinning each of these. Index redirection is done in SQL via REPLACE with quote_ident(), which matches pg_get_indexdef's own output exactly. Tables whose names need quoting (uppercase, reserved words) work as a result, and the single catalog query returns index name, is-primary flag, and pre-rewritten CREATE INDEX statement together. --- .../postgresql/branched/shadow.rb | 53 ++++++- test/shadow_test.rb | 149 ++++++++++++++++++ 2 files changed, 201 insertions(+), 1 deletion(-) diff --git a/lib/active_record/connection_adapters/postgresql/branched/shadow.rb b/lib/active_record/connection_adapters/postgresql/branched/shadow.rb index 346c577..a9ca9d7 100644 --- a/lib/active_record/connection_adapters/postgresql/branched/shadow.rb +++ b/lib/active_record/connection_adapters/postgresql/branched/shadow.rb @@ -36,15 +36,66 @@ def already_shadowed?(table) def create_shadow(table) quoted_branch = @connection.quote_column_name(@branch_schema) quoted_table = @connection.quote_column_name(table) + + clone_structure(quoted_branch, quoted_table) + copy_data(quoted_branch, quoted_table) + clone_indexes(table, quoted_branch, quoted_table) + end + + # Per-row index maintenance during INSERT SELECT dominates the cost + # of shadowing large tables. LIKE ... INCLUDING ALL builds every + # index up front, then the data copy updates each index for every + # row. EXCLUDING INDEXES lets the bulk copy run without index + # overhead; clone_indexes rebuilds them afterwards in a single + # sort pass per index, the standard Postgres bulk-load pattern. + def clone_structure(quoted_branch, quoted_table) @connection.execute(<<~SQL) CREATE TABLE #{quoted_branch}.#{quoted_table} - (LIKE public.#{quoted_table} INCLUDING ALL) + (LIKE public.#{quoted_table} INCLUDING ALL EXCLUDING INDEXES) SQL + end + + def copy_data(quoted_branch, quoted_table) @connection.execute(<<~SQL) INSERT INTO #{quoted_branch}.#{quoted_table} SELECT * FROM public.#{quoted_table} SQL end + + def clone_indexes(table, quoted_branch, quoted_table) + index_metadata(table).each do |index_name, is_primary, branched_indexdef| + @connection.execute(branched_indexdef) + + next unless is_primary + + quoted_index = @connection.quote_column_name(index_name) + @connection.execute(<<~SQL) + ALTER TABLE #{quoted_branch}.#{quoted_table} + ADD CONSTRAINT #{quoted_index} PRIMARY KEY USING INDEX #{quoted_index} + SQL + end + end + + # Redirect pg_get_indexdef output from public to the branch schema. + # The REPLACE uses the same quote_ident() that pg_get_indexdef uses + # to emit identifiers, so the anchor always matches regardless of + # whether the table name needs quoting. + def index_metadata(table) + @connection.select_rows(<<~SQL) + SELECT i.relname, ix.indisprimary, + REPLACE( + pg_get_indexdef(ix.indexrelid), + ' ON public.' || quote_ident(t.relname), + ' ON ' || quote_ident(#{@connection.quote(@branch_schema)}) || '.' || quote_ident(t.relname) + ) + FROM pg_index ix + JOIN pg_class i ON i.oid = ix.indexrelid + JOIN pg_class t ON t.oid = ix.indrelid + JOIN pg_namespace n ON n.oid = t.relnamespace + WHERE n.nspname = 'public' + AND t.relname = #{@connection.quote(table)} + SQL + end end end end diff --git a/test/shadow_test.rb b/test/shadow_test.rb index f7932b8..c88a162 100644 --- a/test/shadow_test.rb +++ b/test/shadow_test.rb @@ -244,4 +244,153 @@ def test_shadow_preserves_data_accessible_via_foreign_key conn.execute("INSERT INTO posts (author_id, title, body) VALUES (#{author_id}, 'world', 'content')") assert_equal 2, conn.select_value("SELECT count(*) FROM posts") end + + # --- Bulk-load shadow preserves every structural detail --- + # + # These tests exist because the shadow now skips INCLUDING INDEXES during + # CREATE TABLE LIKE and rebuilds indexes after the data copy. They pin the + # contract that the branch table is structurally equivalent to public. + + def test_shadow_preserves_primary_key_as_primary_key + conn = reconnect(branch: "feature/pk") + conn.add_column :users, :bio, :string + + is_primary = conn.select_value(<<~SQL) + SELECT ix.indisprimary FROM pg_index ix + JOIN pg_class t ON t.oid = ix.indrelid + JOIN pg_namespace n ON n.oid = t.relnamespace + WHERE n.nspname = 'branch_feature_pk' + AND t.relname = 'users' + AND ix.indisprimary + SQL + assert_equal true, is_primary, + "Shadow must attach the pkey index as a PRIMARY KEY constraint, not just a unique index" + end + + def test_shadow_primary_key_sequence_still_works + conn = reconnect(branch: "feature/pk-insert") + conn.add_column :users, :bio, :string + + conn.execute("INSERT INTO users (name, legacy, bio) VALUES ('carol', 'old', 'hi')") + inserted_id = conn.select_value("SELECT id FROM users WHERE name = 'carol'") + refute_nil inserted_id, "Inserts into the shadow must auto-assign an id" + end + + def test_shadow_preserves_unique_index + conn = connect(branch: "main") + conn.execute("CREATE UNIQUE INDEX users_name_uniq ON public.users (name)") + + conn = reconnect(branch: "feature/uniq-idx") + conn.add_column :users, :bio, :string + + assert index_exists_in_schema?(conn, "branch_feature_uniq_idx", "users_name_uniq") + is_unique = conn.select_value(<<~SQL) + SELECT ix.indisunique FROM pg_index ix + JOIN pg_class i ON i.oid = ix.indexrelid + JOIN pg_namespace n ON n.oid = i.relnamespace + WHERE n.nspname = 'branch_feature_uniq_idx' AND i.relname = 'users_name_uniq' + SQL + assert_equal true, is_unique, "Unique index must be rebuilt as unique" + + assert_raises(ActiveRecord::RecordNotUnique) do + conn.execute("INSERT INTO users (name, legacy, bio) VALUES ('alice', 'new', 'dup')") + end + end + + def test_shadow_preserves_non_unique_index + conn = connect(branch: "main") + conn.execute("CREATE INDEX users_legacy_idx ON public.users (legacy)") + + conn = reconnect(branch: "feature/plain-idx") + conn.add_column :users, :bio, :string + + assert index_exists_in_schema?(conn, "branch_feature_plain_idx", "users_legacy_idx") + end + + def test_shadow_preserves_check_constraint + conn = connect(branch: "main") + conn.execute("ALTER TABLE public.orders ADD CONSTRAINT orders_amount_positive CHECK (amount > 0)") + + conn = reconnect(branch: "feature/check") + conn.add_column :orders, :note, :string + + assert check_constraint_exists_in_schema?(conn, "branch_feature_check", "orders", "orders_amount_positive") + assert_raises(ActiveRecord::StatementInvalid) do + conn.execute("INSERT INTO orders (status, amount, note) VALUES ('bad', -1, 'x')") + end + end + + def test_shadow_preserves_not_null_and_default + conn = connect(branch: "main") + conn.execute("ALTER TABLE public.orders ALTER COLUMN status SET NOT NULL") + conn.execute("ALTER TABLE public.orders ALTER COLUMN status SET DEFAULT 'pending'") + + conn = reconnect(branch: "feature/default") + conn.add_column :orders, :note, :string + + is_not_null = conn.select_value(<<~SQL) + SELECT attnotnull FROM pg_attribute a + JOIN pg_class t ON t.oid = a.attrelid + JOIN pg_namespace n ON n.oid = t.relnamespace + WHERE n.nspname = 'branch_feature_default' + AND t.relname = 'orders' + AND a.attname = 'status' + SQL + assert_equal true, is_not_null, "NOT NULL must survive shadow" + + conn.execute("INSERT INTO orders (amount, note) VALUES (50, 'uses-default')") + status = conn.select_value("SELECT status FROM orders WHERE note = 'uses-default'") + assert_equal "pending", status, "Default must survive shadow" + end + + def test_shadow_preserves_multi_column_index + conn = connect(branch: "main") + conn.execute("CREATE INDEX orders_status_amount ON public.orders (status, amount)") + + conn = reconnect(branch: "feature/multi-col") + conn.add_column :orders, :note, :string + + columns = conn.select_values(<<~SQL) + SELECT a.attname FROM pg_index ix + JOIN pg_class i ON i.oid = ix.indexrelid + JOIN pg_namespace n ON n.oid = i.relnamespace + JOIN pg_attribute a ON a.attrelid = ix.indrelid AND a.attnum = ANY(ix.indkey) + WHERE n.nspname = 'branch_feature_multi_col' AND i.relname = 'orders_status_amount' + ORDER BY array_position(ix.indkey, a.attnum) + SQL + assert_equal %w[status amount], columns, "Column order in multi-column index must be preserved" + end + + def test_shadow_preserves_functional_and_partial_indexes + conn = connect(branch: "main") + # Functional and partial indexes give pg_get_indexdef more complex output — + # exactly the case where a naive " ON public." string replacement could go + # wrong. The rewrite must still target only the table qualifier. + conn.execute("CREATE INDEX users_lower_name ON public.users (lower(name))") + conn.execute("CREATE INDEX users_non_legacy ON public.users (name) WHERE legacy IS NULL") + + conn = reconnect(branch: "feature/fn-idx") + conn.add_column :users, :bio, :string + + assert index_exists_in_schema?(conn, "branch_feature_fn_idx", "users_lower_name") + assert index_exists_in_schema?(conn, "branch_feature_fn_idx", "users_non_legacy") + end + + def test_shadow_preserves_indexes_on_quoted_table_name + # A table whose name happens to need SQL quoting (capital letters, reserved + # word, etc.) surfaces quoting bugs in the index-rewrite path. pg_get_indexdef + # quotes only when necessary; the adapter's Ruby-side quoting always does. + # Mismatched expectations here made the first take on this rewrite fail. + conn = connect(branch: "main") + conn.execute('CREATE TABLE public."User" (id serial PRIMARY KEY, email varchar)') + conn.execute('CREATE INDEX user_email_idx ON public."User" (email)') + conn.execute("INSERT INTO public.\"User\" (email) VALUES ('a@x.com')") + + conn = reconnect(branch: "feature/quoted-name") + conn.add_column "User", :bio, :string + + assert table_exists_in_schema?(conn, "branch_feature_quoted_name", "User") + assert index_exists_in_schema?(conn, "branch_feature_quoted_name", "user_email_idx") + assert_equal 1, conn.select_value('SELECT count(*) FROM "User"') + end end From 6cb472ba148a72cf047a6220c0aa627888631e94 Mon Sep 17 00:00:00 2001 From: mhenrixon Date: Fri, 17 Apr 2026 17:25:17 +0200 Subject: [PATCH 2/2] Add unlogged_branches option: shadow tables as UNLOGGED MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Branch schemas are disposable by design — db:branch:reset rebuilds them from public — so the crash-loss semantics of UNLOGGED are a natural fit for dev/test. UNLOGGED skips WAL for writes, which reduces disk pressure and cuts shadow time further on slower storage. Opt-in via unlogged_branches: true in database.yml. Default stays false to preserve current behavior; users can flip it on once they've confirmed nothing in their flow depends on WAL'd shadow tables. Benchmarks on a widgets table with 1 PK + 5 indexes (builds on the bulk-load optimisation in the previous change): 3M rows, logged bulk: 17.35s 3M rows, unlogged bulk: 14.62s (additional 1.19x) Speedup is modest on local SSD because WAL isn't the bottleneck there; larger tables, slower storage, and concurrent workloads widen the gap. Caveat: a foreign key added on a feature branch targeting a public-only table will fail when the shadow is UNLOGGED — Postgres forbids logged/unlogged FK pairs. Shadow the reference first or turn the option off. This does not affect any FKs carried through from public since LIKE ... INCLUDING CONSTRAINTS already excludes FK constraints. --- README.md | 10 +++ .../postgresql/branched/adapter.rb | 8 ++- .../postgresql/branched/shadow.rb | 10 ++- test/shadow_test.rb | 61 +++++++++++++++++++ test/test_helper.rb | 21 +++++-- 5 files changed, 102 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 5d9fa40..d1d9054 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,7 @@ development: adapter: postgresql_branched database: myapp_development primary_branch: main # default, can be 'master', 'trunk', etc. + unlogged_branches: false # default; set to true to shadow tables UNLOGGED ``` The adapter needs a branch name. Resolution order: @@ -151,6 +152,15 @@ All standard PostgreSQL connection parameters work as normal (`host`, `port`, `u **Do not set `schema_search_path` in database.yml** — it conflicts with the adapter's `search_path` management. +### Unlogged branch shadows + +Set `unlogged_branches: true` to create shadow tables as Postgres `UNLOGGED`. WAL is skipped for shadow writes, which reduces disk pressure on large prod-restored dev databases and shortens shadow time on slow storage. + +Branch schemas are disposable by design — `db:branch:reset` rebuilds them from `public` — so the crash-loss semantics of UNLOGGED are a good match for dev/test. Two caveats: + +- A foreign key added on a feature branch (`add_foreign_key :shadowed, :public_only_table`) fails when the shadow is UNLOGGED and the referenced table is still LOGGED in `public`. Either shadow the reference first, or turn the option off. +- If Postgres crashes, all shadow tables are truncated and you re-run `db:branch:reset` + `db:migrate`. No worse than switching branches. + ## Limitations - **Rails + PostgreSQL only** — uses PostgreSQL schemas and `search_path` diff --git a/lib/active_record/connection_adapters/postgresql/branched/adapter.rb b/lib/active_record/connection_adapters/postgresql/branched/adapter.rb index 4fcb8fc..cbde7d2 100644 --- a/lib/active_record/connection_adapters/postgresql/branched/adapter.rb +++ b/lib/active_record/connection_adapters/postgresql/branched/adapter.rb @@ -31,7 +31,13 @@ class Adapter < ActiveRecord::ConnectionAdapters::PostgreSQLAdapter def initialize(...) super @branch_manager = BranchManager.new(self, @config) - @shadow = Shadow.new(self, @branch_manager.branch_schema) unless @branch_manager.primary_branch? + return if @branch_manager.primary_branch? + + @shadow = Shadow.new( + self, + @branch_manager.branch_schema, + unlogged: @config.fetch(:unlogged_branches, false) + ) end def configure_connection diff --git a/lib/active_record/connection_adapters/postgresql/branched/shadow.rb b/lib/active_record/connection_adapters/postgresql/branched/shadow.rb index a9ca9d7..3cca318 100644 --- a/lib/active_record/connection_adapters/postgresql/branched/shadow.rb +++ b/lib/active_record/connection_adapters/postgresql/branched/shadow.rb @@ -3,9 +3,10 @@ module ConnectionAdapters module PostgreSQL module Branched class Shadow - def initialize(connection, branch_schema) + def initialize(connection, branch_schema, unlogged: false) @connection = connection @branch_schema = branch_schema + @unlogged = unlogged end def call(table_name) @@ -48,9 +49,14 @@ def create_shadow(table) # row. EXCLUDING INDEXES lets the bulk copy run without index # overhead; clone_indexes rebuilds them afterwards in a single # sort pass per index, the standard Postgres bulk-load pattern. + # + # UNLOGGED (when enabled) skips WAL for both the bulk copy and + # subsequent writes. Branch schemas are disposable by design — + # `db:branch:reset` rebuilds them from public — so the crash-loss + # semantics of UNLOGGED are a good fit for dev/test. def clone_structure(quoted_branch, quoted_table) @connection.execute(<<~SQL) - CREATE TABLE #{quoted_branch}.#{quoted_table} + CREATE #{"UNLOGGED " if @unlogged}TABLE #{quoted_branch}.#{quoted_table} (LIKE public.#{quoted_table} INCLUDING ALL EXCLUDING INDEXES) SQL end diff --git a/test/shadow_test.rb b/test/shadow_test.rb index c88a162..20c78f5 100644 --- a/test/shadow_test.rb +++ b/test/shadow_test.rb @@ -393,4 +393,65 @@ def test_shadow_preserves_indexes_on_quoted_table_name assert index_exists_in_schema?(conn, "branch_feature_quoted_name", "user_email_idx") assert_equal 1, conn.select_value('SELECT count(*) FROM "User"') end + + # --- unlogged_branches option --- + # + # pg_class.relpersistence values: + # 'p' = permanent (default), 'u' = unlogged, 't' = temporary + + def test_shadow_is_logged_by_default + conn = reconnect(branch: "feature/logged") + conn.add_column :users, :bio, :string + + assert_equal "p", relpersistence(conn, "branch_feature_logged", "users") + end + + def test_shadow_is_unlogged_when_enabled + conn = reconnect(branch: "feature/unlogged", unlogged_branches: true) + conn.add_column :users, :bio, :string + + assert_equal "u", relpersistence(conn, "branch_feature_unlogged", "users") + end + + def test_unlogged_shadow_still_has_primary_key_and_data + conn = reconnect(branch: "feature/unlogged-pk", unlogged_branches: true) + conn.add_column :users, :bio, :string + + assert_equal 2, conn.select_value("SELECT count(*) FROM users"), + "Unlogged shadow must still copy data" + is_primary = conn.select_value(<<~SQL) + SELECT ix.indisprimary FROM pg_index ix + JOIN pg_class t ON t.oid = ix.indrelid + JOIN pg_namespace n ON n.oid = t.relnamespace + WHERE n.nspname = 'branch_feature_unlogged_pk' + AND t.relname = 'users' + AND ix.indisprimary + SQL + assert_equal true, is_primary, "Unlogged shadow must keep the PK designation" + end + + def test_unlogged_indexes_inherit_unlogged_status + conn = connect(branch: "main") + conn.execute("CREATE INDEX users_name_idx ON public.users (name)") + + conn = reconnect(branch: "feature/unlogged-idx", unlogged_branches: true) + conn.add_column :users, :bio, :string + + index_persistence = conn.select_value(<<~SQL) + SELECT c.relpersistence FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'branch_feature_unlogged_idx' + AND c.relname = 'users_name_idx' + SQL + assert_equal "u", index_persistence, + "Indexes on unlogged tables must themselves be unlogged" + end + + def test_unlogged_option_is_ignored_on_primary_branch + conn = reconnect(branch: "main", unlogged_branches: true) + conn.add_column :users, :bio, :string + + assert_equal "p", relpersistence(conn, "public", "users"), + "Primary branch must never create UNLOGGED tables — public holds the canonical data" + end end diff --git a/test/test_helper.rb b/test/test_helper.rb index a3d7c23..c971b81 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -24,19 +24,30 @@ def teardown_test_database ENV.delete("PGBRANCH") end - def connect(branch:, primary_branch: "main") + def connect(branch:, primary_branch: "main", unlogged_branches: nil) ENV["PGBRANCH"] = branch - ActiveRecord::Base.establish_connection( + config = { adapter: "postgresql_branched", database: DATABASE_NAME, primary_branch: primary_branch - ) + } + config[:unlogged_branches] = unlogged_branches unless unlogged_branches.nil? + ActiveRecord::Base.establish_connection(config) ActiveRecord::Base.lease_connection end - def reconnect(branch:, primary_branch: "main") + def reconnect(branch:, primary_branch: "main", unlogged_branches: nil) ActiveRecord::Base.connection_pool.disconnect! - connect(branch: branch, primary_branch: primary_branch) + connect(branch: branch, primary_branch: primary_branch, unlogged_branches: unlogged_branches) + end + + def relpersistence(connection, schema_name, table_name) + connection.select_value(<<~SQL) + SELECT relpersistence FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = #{connection.quote(schema_name)} + AND c.relname = #{connection.quote(table_name)} + SQL end def schema_exists?(connection, schema_name)