From 3f94c68f9fc9ef56722ab5db56c5c1675fdb0c55 Mon Sep 17 00:00:00 2001 From: Rob Galanakis Date: Thu, 25 Jun 2026 09:28:34 -0700 Subject: [PATCH 1/7] Analytics: Add charge.incurred_at We need to know when the charge was incurred by the user, not when Suma processed it, for example if we have to do a historical import. --- Makefile | 3 ++ db/migrations/112_charge_analytics_created.rb | 9 +++++ lib/suma/analytics/charge.rb | 33 +++++++++++-------- lib/suma/analytics/model.rb | 12 +++---- spec/suma/analytics_spec.rb | 23 ++++++------- 5 files changed, 48 insertions(+), 32 deletions(-) create mode 100644 db/migrations/112_charge_analytics_created.rb diff --git a/Makefile b/Makefile index d7b67fd96..99f52bda6 100644 --- a/Makefile +++ b/Makefile @@ -134,6 +134,9 @@ analytics-reimport: @bundle exec rake analytics:truncate @bundle exec rake analytics:import +analytics-reimport-production: + heroku run:detached bundle exec rake analytics:import --app $(production_app) + take-production-db-snapshot: heroku pg:backups:capture --app $(production_app) diff --git a/db/migrations/112_charge_analytics_created.rb b/db/migrations/112_charge_analytics_created.rb new file mode 100644 index 000000000..fb50c74b5 --- /dev/null +++ b/db/migrations/112_charge_analytics_created.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +Sequel.migration do + change do + alter_table(Sequel[:analytics][:charges]) do + add_column :incurred_at, :timestamptz + end + end +end diff --git a/lib/suma/analytics/charge.rb b/lib/suma/analytics/charge.rb index 7d67c91ee..1cb97d805 100644 --- a/lib/suma/analytics/charge.rb +++ b/lib/suma/analytics/charge.rb @@ -7,19 +7,26 @@ class Suma::Analytics::Charge < Suma::Analytics::Model(Sequel[:analytics][:charg destroy_from Suma::Charge - denormalize Suma::Charge, with: [ - [:charge_id, :id], - :opaque_id, - :created_at, - :member_id, - [:order_id, :commerce_order_id], - [:trip_id, :mobility_trip_id], - :undiscounted_subtotal, - :discounted_subtotal, - :discount_amount, - [:cash_paid, :cash_paid_from_ledger], - [:noncash_paid, :noncash_paid_from_ledger], - ] + denormalize Suma::Charge, with: :denormalize_charge + + def self.denormalize_charge(charge) + # one or the other must be set + incurred_at = charge.mobility_trip&.ended_at || charge.commerce_order.created_at + return { + charge_id: charge.id, + opaque_id: charge.opaque_id, + created_at: charge.created_at, + member_id: charge.member_id, + order_id: charge.commerce_order_id, + trip_id: charge.mobility_trip_id, + incurred_at:, + undiscounted_subtotal: charge.undiscounted_subtotal, + discounted_subtotal: charge.discounted_subtotal, + discount_amount: charge.discount_amount, + cash_paid: charge.cash_paid_from_ledger, + noncash_paid: charge.noncash_paid_from_ledger, + } + end end # Table: analytics.charges diff --git a/lib/suma/analytics/model.rb b/lib/suma/analytics/model.rb index 57d5dd6c8..ea120008f 100644 --- a/lib/suma/analytics/model.rb +++ b/lib/suma/analytics/model.rb @@ -63,12 +63,12 @@ def unique_key(sym=nil) # - A +Proc+, called with the transactional model instance. # - An +Array+, which is a shorthand for denormalization. Each item in the array is one of: # - A +Symbol+, like `:name`, - # which would add a cell for `name=model.name`. - # - A tuple of symbols, like `[:id, :member_id]`, - # which would add a cell for `member_id=model_id`. - # - A tuple of a symbol and a symbol array, like `[:id, [:member, :id]]`, - # which would add a cell for `member_id=member.id`. - # - A tuple of a symbol and proc, like `[:email, ->(m) { m.email.upcase }]`, + # which would add a cell for `name=tmodel.name`. + # - A tuple of symbols, like `[:member_id, :id]`, + # which would add a cell for `member_id=tmodel.id`. + # - A tuple of a symbol and a symbol array, like `[:member_id, [:member, :id]]`, + # which would add a cell for `member_id=tmodel.member.id`. + # - A tuple of a symbol and proc, like `[:email, ->(tmodel) { tmodel.email.upcase }]`, # called with the model instance, which would add a cell like `email='A@B.C'`. def denormalize(transactional_model_class, with:) self.denormalizers[transactional_model_class] = with diff --git a/spec/suma/analytics_spec.rb b/spec/suma/analytics_spec.rb index 5e17e4893..cafb6de6e 100644 --- a/spec/suma/analytics_spec.rb +++ b/spec/suma/analytics_spec.rb @@ -139,28 +139,25 @@ end describe "Charge" do - it "denormalizes from order charges" do - o = Suma::Fixtures.charge.create - Suma::Analytics.upsert_from_transactional_model(o) - expect(Suma::Analytics::Charge.dataset.all).to contain_exactly( - include(charge_id: o.id), - ) - end - - it "denormalizes from order charges" do - o = Suma::Fixtures.charge.create + it "denormalizes from trip charges" do + t = trunc_time(1.hour.ago) + trip = Suma::Fixtures.mobility_trip.ended.create(ended_at: t) + o = Suma::Fixtures.charge.create(mobility_trip: trip) expect(o.mobility_trip).to_not be_nil Suma::Analytics.upsert_from_transactional_model(o) expect(Suma::Analytics::Charge.dataset.all).to contain_exactly( - include(charge_id: o.id), + include(charge_id: o.id, trip_id: trip.id, incurred_at: t), ) end it "denormalizes from order charges" do - o = Suma::Fixtures.charge.create(commerce_order: Suma::Fixtures.order.create) + order = Suma::Fixtures.order.create + t = trunc_time(1.hour.ago) + order.update(created_at: t) + o = Suma::Fixtures.charge.create(commerce_order: order) Suma::Analytics.upsert_from_transactional_model(o) expect(Suma::Analytics::Charge.dataset.all).to contain_exactly( - include(charge_id: o.id), + include(charge_id: o.id, order_id: order.id, incurred_at: t), ) end end From fa2943111cc21a99c08b2a69218751fb1b744e48 Mon Sep 17 00:00:00 2001 From: Rob Galanakis Date: Thu, 25 Jun 2026 09:44:07 -0700 Subject: [PATCH 2/7] Rename prepare_prod_db_for_testing It's only valid for local dev, since it contains potentially sensitive data; it should not be used in places like staging. --- Makefile | 2 +- lib/suma/tasks/release.rb | 7 +++++-- spec/tasks/release_spec.rb | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 99f52bda6..2dc39f0bc 100644 --- a/Makefile +++ b/Makefile @@ -152,7 +152,7 @@ restore-db-from-dump: @PGPASSWORD=suma psql postgres://suma:suma@localhost:22005/suma -c "CREATE SCHEMA IF NOT EXISTS heroku_ext; ALTER DATABASE suma SET search_path TO public,heroku_ext;" PGPASSWORD=suma pg_restore --clean --no-acl --no-owner -h 127.0.0.1 -p 22005 -U suma -d suma temp/latest.dump || true @PGPASSWORD=suma psql postgres://suma:suma@localhost:22005/suma -c "ALTER EXTENSION citext SET SCHEMA public" - @bundle exec rake release:prepare_prod_db_for_testing + @bundle exec rake release:prepare_prod_db_for_local @./bin/notify "Finished restoring database from production" diff --git a/lib/suma/tasks/release.rb b/lib/suma/tasks/release.rb index ba02682c3..3aa579fb9 100644 --- a/lib/suma/tasks/release.rb +++ b/lib/suma/tasks/release.rb @@ -20,8 +20,10 @@ def initialize end namespace :release do - desc "Set every user password to #{PASSWORD}." - task :prepare_prod_db_for_testing do + desc "Prepare the current database dump for local development by " \ + "setting every user password to #{PASSWORD} and " \ + "undeleting the admin@lithic.tech user." + task :prepare_prod_db_for_local do # Do NOT use load_app. We may have local migrations not applied to the dump, # and we'll error trying to load those models. require "suma/member" @@ -39,6 +41,7 @@ def initialize end end + desc "Randomize all member passwords." task :randomize_passwords do Suma.load_app? Suma::Member.exclude(email: nil).each do |m| diff --git a/spec/tasks/release_spec.rb b/spec/tasks/release_spec.rb index 31c3420e1..8bb040c39 100644 --- a/spec/tasks/release_spec.rb +++ b/spec/tasks/release_spec.rb @@ -19,12 +19,12 @@ end end - describe "prepare_prod_db_for_testing", db: :no_transaction do + describe "prepare_prod_db_for_local", db: :no_transaction do it "cleans passwords, stripe json, and un-deletes superadmin" do m = Suma::Fixtures.member.create(password: SecureRandom.hex(20), stripe_customer_json: "{}") expect(m.authenticate?("suma1234")).to be(false) admin = Suma::Fixtures.member.create(email: "admin@lithic.tech", soft_deleted_at: Time.now) - invoke_rake_task("release:prepare_prod_db_for_testing") + invoke_rake_task("release:prepare_prod_db_for_local") expect(m.refresh.authenticate?("Password1!")).to be(true) expect(m.stripe_customer_json).to be_nil expect(admin.refresh).to_not be_soft_deleted From 34a96c1b3d23bd66d4b95c975b0e28b4444c224e Mon Sep 17 00:00:00 2001 From: Rob Galanakis Date: Thu, 25 Jun 2026 16:25:37 -0700 Subject: [PATCH 3/7] Support secure restores to staging This adds a new rake task that can be used to restore a production dump to a staging database, removing all non-admin member data. This ensures that even if someone makes it to staging or somehow grabs a copy of the DB, no member data is included. --- Makefile | 28 ++-- bin/notify | 14 +- lib/suma/analytics/model.rb | 2 + lib/suma/postgres.rb | 11 ++ lib/suma/postgres/model.rb | 2 + lib/suma/tasks/db.rb | 13 +- lib/suma/tasks/release.rb | 298 ++++++++++++++++++++++++++++++++++++ lib/suma/webhookdb/model.rb | 2 + spec/suma/postgres_spec.rb | 12 ++ spec/tasks/db_spec.rb | 4 +- spec/tasks/release_spec.rb | 12 ++ 11 files changed, 370 insertions(+), 28 deletions(-) diff --git a/Makefile b/Makefile index 2dc39f0bc..ddd84210f 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,8 @@ production_app:=suma-production OUT ?= - MESSAGE_LANG ?= MESSAGE_TRANSPORT ?= +DBDUMP = temp/latest.dump +DBURL_LOCAL = postgres://suma:suma@localhost:22005/suma install: bundle install @@ -102,7 +104,7 @@ annotate: RACK_ENV=test LOG_LEVEL=info bundle exec rake annotate psql: cmd-exists-pgcli - pgcli postgres://suma:suma@localhost:22005/suma + pgcli $(DBURL_LOCAL) psql-test: cmd-exists-pgcli pgcli postgres://suma:suma@localhost:22006/suma_test psql-%: cmd-exists-pgcli @@ -142,26 +144,28 @@ take-production-db-snapshot: download-production-dump: @mkdir -p temp - @rm -f latest.dump - heroku pg:backups:download --app $(production_app) - @mv latest.dump temp/latest.dump + @rm -f $(DBDUMP) + heroku pg:backups:download -o $(DBDUMP) --app $(production_app) + @./bin/notify "Downloaded production dump" -restore-db-from-dump: +restore-dump-for-local-db: @bundle exec rake db:drop_tables - @mkdir -p temp - @PGPASSWORD=suma psql postgres://suma:suma@localhost:22005/suma -c "CREATE SCHEMA IF NOT EXISTS heroku_ext; ALTER DATABASE suma SET search_path TO public,heroku_ext;" - PGPASSWORD=suma pg_restore --clean --no-acl --no-owner -h 127.0.0.1 -p 22005 -U suma -d suma temp/latest.dump || true - @PGPASSWORD=suma psql postgres://suma:suma@localhost:22005/suma -c "ALTER EXTENSION citext SET SCHEMA public" + @PGPASSWORD=suma psql $(DBURL_LOCAL) -c "CREATE SCHEMA IF NOT EXISTS heroku_ext; ALTER DATABASE suma SET search_path TO public,heroku_ext;" + PGPASSWORD=suma pg_restore --clean --if-exists --no-acl --no-owner -d $(DBURL_LOCAL) $(DBDUMP) || true + @PGPASSWORD=suma psql $(DBURL_LOCAL) -c "ALTER EXTENSION citext SET SCHEMA public" @bundle exec rake release:prepare_prod_db_for_local - @./bin/notify "Finished restoring database from production" + @./bin/notify "Finished restoring database from dump" +restore-dump-for-staging-db: + @bundle exec rake release:restore_staging_db_from_dump[$(DBDUMP)] + @./bin/notify "Finished restoring staging from dump" reinit-db-from-dump: docker compose down -v docker compose up -d sleep 5 - make restore-db-from-dump - @echo "Remember to migrate your test DB before running tests by running 'make migrate-test'" + make restore-dump-for-local-db + make migrate-test build-webapp: @bundle exec rake frontend:build_webapp diff --git a/bin/notify b/bin/notify index 17aed4ea0..f6731bb57 100755 --- a/bin/notify +++ b/bin/notify @@ -1,4 +1,16 @@ #!/usr/bin/env bash set -e -osascript -e "display notification \"$1\" with title \"Suma\"" +MESSAGE="$1" +TITLE="Suma" + +case "$(uname -s)" in + Darwin) + osascript -e "display notification \"$MESSAGE\" with title \"$TITLE\"" + ;; + Linux) + if command -v notify-send >/dev/null 2>&1; then + notify-send "$TITLE" "$MESSAGE" + fi + ;; +esac diff --git a/lib/suma/analytics/model.rb b/lib/suma/analytics/model.rb index ea120008f..6aa3c0eab 100644 --- a/lib/suma/analytics/model.rb +++ b/lib/suma/analytics/model.rb @@ -36,6 +36,8 @@ class RowMismatch < StandardError; end end end + def self.schema = :analytics + def self.inherited(subclass) super subclass.extend(ClassMethods) diff --git a/lib/suma/postgres.rb b/lib/suma/postgres.rb index 9a290d5e0..ff29c6287 100644 --- a/lib/suma/postgres.rb +++ b/lib/suma/postgres.rb @@ -235,6 +235,17 @@ def self.load_models end end + # Drop all tables in the database for all model superclasses. + def self.drop_all_tables + self.load_superclasses + self.model_superclasses.reject(&:read_only?).each do |sc| + schemaname = sc.schema.to_s + sc.db[:pg_tables].where(schemaname:).each do |tbl| + sc.db.execute("DROP TABLE #{schemaname}.#{tbl[:tablename]} CASCADE") + end + end + end + # Return 'Time.now' as an expression suitable for Sequel/SQL. # In some cases (like range @> expressions) you need to cast to a timestamptz explicitly, # the implicit cast isn't enough. diff --git a/lib/suma/postgres/model.rb b/lib/suma/postgres/model.rb index 1eee8ca9f..a3ae8eceb 100644 --- a/lib/suma/postgres/model.rb +++ b/lib/suma/postgres/model.rb @@ -93,6 +93,8 @@ class Suma::Postgres::Model end end + def self.schema = :public + def self.extensions return [ "citext", diff --git a/lib/suma/tasks/db.rb b/lib/suma/tasks/db.rb index d8242965a..d00e05cb0 100644 --- a/lib/suma/tasks/db.rb +++ b/lib/suma/tasks/db.rb @@ -11,18 +11,7 @@ def initialize desc "Drop all tables in the public schema." task :drop_tables do require "suma/postgres" - Suma::Postgres.load_superclasses - # We cannot use load_models to get the schemas they use, in case the models cannot load correctly. - # So just hard-code the known schemas that we use. - schemas = ["public", "analytics"] - Suma::Postgres.model_superclasses.reject(&:read_only?).each do |sc| - next if sc == Suma::Webhookdb::Model && Suma::RACK_ENV != "test" - schemas.each do |schemaname| - sc.db[:pg_tables].where(schemaname:).each do |tbl| - Suma::Tasks::DB.exec(sc.db, "DROP TABLE #{schemaname}.#{tbl[:tablename]} CASCADE") - end - end - end + Suma::Postgres.drop_all_tables end desc "Remove all data from application schemas" diff --git a/lib/suma/tasks/release.rb b/lib/suma/tasks/release.rb index 3aa579fb9..55e521c46 100644 --- a/lib/suma/tasks/release.rb +++ b/lib/suma/tasks/release.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require "rake/tasklib" +require "sequel" require "suma/tasks" require "suma/tasks/db" @@ -50,6 +51,303 @@ def initialize $stdout << "#{m.email}: #{pw}\n" end end + + desc "Given a PG dump at the given path, restore platform (non-member) data." + task :restore_staging_db_from_dump, [:path] do |_, args| + # This routine is pretty fun. Here's what we do: + # - First, reset the DB to a clean state by dropping all tables, + # and then reloading schemas (not data) from the dump. + # - We can safely load the app at this point since schemas are correct. + # - Convert all the ON RESTRICT (or NO ACTION, same thing) FKs to CASCADE. + # - Load in all data (we could do this before the CASCADE, but it's easier to split it up for dev purposes). + # - Truncate some sensitive tables we want to remove entirely. + # - Delete all members who are not admins. The DELETE will cascade and clean up all associated data. + # - Remove some selective data, like now-unused addresses. + anon = StagingAnonymizer.new(args.fetch(:path)) + require "suma/postgres" + anon.drop_all_tables + anon.run_pgrestore("--clean --if-exists --schema-only") + + Suma.load_app? + anon.each_fk_constraint do |schema, tbl, con| + anon.cascade_constraint(schema, tbl, con) + end + + anon.run_pgrestore("--data-only --disable-triggers") + anon.truncate_analytics + + anon.tables_to_truncate.each do |schema, tables| + tables.each { |tbl| anon.truncate(schema, tbl) } + end + anon.delete_selective + + anon.each_fk_constraint do |schema, tbl, con| + anon.restore_constraint(schema, tbl, con) + end + end + end + end + + class StagingAnonymizer + def initialize(dump) + @dump = dump + @dburl = Suma::Postgres::Model.uri + @db = Suma::Member.db + @constraint_originals = {} + end + + def drop_all_tables = self.class.drop_all_tables + + def self.drop_all_tables + Suma::Postgres.drop_all_tables + end + + def run_pgrestore(params) = self.class.run_pgrestore(%(--no-acl --no-owner #{params} -d "#{@dburl}" "#{@dump}")) + + def self.run_pgrestore(argstr) + `pg_restore #{argstr}` + end + + def _fetch_constraint_def(schema, table, constraint_name) + result = @db.fetch(<<~SQL, schema.to_s, table.to_s, constraint_name.to_s).all + SELECT pg_get_constraintdef(c.oid) AS def + FROM pg_constraint c + JOIN pg_class t ON t.oid = c.conrelid + JOIN pg_namespace n ON n.oid = t.relnamespace + WHERE n.nspname = ? AND t.relname = ? AND c.conname = ? + SQL + raise "Constraint not found: #{schema}.#{table}.#{constraint_name}" if result.empty? + result[0][:def] + end + + def cascade_constraint(schema, table, name) + original_def = _fetch_constraint_def(schema, table, name) + @constraint_originals[[schema, table, name]] = original_def + + cascade_def = original_def.sub(/ON DELETE \w+(\s+\w+)?/, "").strip + cascade_def = "#{cascade_def} ON DELETE CASCADE" + + # puts "Flipping #{schema}.#{table}.#{name} to CASCADE" + @db.execute(%(ALTER TABLE "#{schema}"."#{table}" DROP CONSTRAINT "#{name}")) + @db.execute(%(ALTER TABLE "#{schema}"."#{table}" ADD CONSTRAINT "#{name}" #{cascade_def})) + end + + def restore_constraint(schema, table, name) + original_def = @constraint_originals[[schema, table, name]] + # puts "Restoring #{schema}.#{table}.#{name}" + @db.execute(%(ALTER TABLE "#{schema}"."#{table}" DROP CONSTRAINT "#{name}")) + @db.execute(%(ALTER TABLE "#{schema}"."#{table}" ADD CONSTRAINT "#{name}" #{original_def})) + end + + def truncate(schema, tbl) + @db.execute("DELETE FROM #{schema}.#{tbl} CASCADE") + end + + def each_fk_constraint + @_all_fks ||= self._all_fks + @_all_fks.each do |schema, tables| + tables.each do |tbl, cons| + cons.each do |con| + yield schema, tbl, con + end + end + end + end + + # Return the tables and their FK constraint names to switch to CASCADE, + # and then back to RESTRICT. + def _all_fks + d1 = self._find_all_join_table_fks + d2 = self._model_fks + d1.deep_merge!(d2) + return d1 + end + + def _find_all_join_table_fks + join_tables = [] + Sequel::Model.descendants.reject(&:anonymous?).each do |model| + model.association_reflections.each_value do |opts| + join_tables << opts[:join_table] if opts[:join_table] + end + end + constraint_infos = @db.fetch(<<~SQL, table_names: join_tables.map(&:to_s)).all + SELECT + n.nspname AS schema, + t.relname AS table_name, + c.conname AS constraint_name + FROM pg_constraint c + JOIN pg_class t ON t.oid = c.conrelid + JOIN pg_namespace n ON n.oid = t.relnamespace + WHERE c.contype = 'f' + AND t.relname IN :table_names + ORDER BY t.relname, c.conname + SQL + + result = {} + constraint_infos.each do |row| + tblmap = result[row[:schema]] ||= {} + fks = tblmap[row[:table_name]] ||= [] + fks << row[:constraint_name] + end + return result + end + + # This list was figured out by finding all "# Foreign key constraints:", + # and then grabbing the FK constraints that should be cascaded on delete. + def _model_fks + return { + public: { + images: [ + :images_mobility_trip_id_fkey, + ], + charges: [ + :charges_member_id_fkey, + ], + members: [ + :members_legal_entity_id_fkey, + ], + anon_proxy_vendor_account_messages: [ + :anon_proxy_vendor_account_messages_outbound_delivery_id_fkey, + ], + commerce_carts: [ # TODO: migration + :commerce_carts_member_id_fkey, + ], + commerce_cart_items: [ # TODO: migration + :commerce_cart_items_cart_id_fkey, + :commerce_cart_items_product_id_fkey, + ], + commerce_checkouts: [ + :commerce_checkouts_bank_account_id_fkey, + :commerce_checkouts_card_id_fkey, + :commerce_checkouts_cart_id_fkey, + ], + commerce_checkout_items: [ + :commerce_checkout_items_cart_item_id_fkey, + ], + commerce_orders: [ + :commerce_orders_checkout_id_fkey, + ], + commerce_order_audit_logs: [ + :commerce_order_audit_logs_order_id_fkey, # TODO: migration + ], + marketing_sms_dispatches: [ + :marketing_sms_dispatches_member_id_fkey, + ], + member_reset_codes: [ + :member_reset_codes_message_delivery_id_fkey, + ], + message_preferences: [ + :message_preferences_member_id_fkey, + ], + mobility_trips: [ + :mobility_trips_member_id_fkey, + ], + organization_memberships: [ + :organization_memberships_member_id_fkey, + ], + organization_membership_verifications: [ + :organization_membership_verifications_membership_id_fkey, # TODO: migrate + ], + payment_accounts: [ + :payment_accounts_member_id_fkey, + ], + payment_bank_accounts: [ + :bank_accounts_legal_entity_id_fkey, + ], + payment_cards: [ + :payment_cards_legal_entity_id_fkey, + ], + payment_off_platform_strategies: [ + :payment_off_platform_strategies_created_by_id_fkey, + ], + payment_book_transactions: [ + :payment_book_transactions_originating_ledger_id_fkey, + :payment_book_transactions_receiving_ledger_id_fkey, + ], + payment_funding_transactions: [ + :payment_funding_transactions_fake_strategy_id_fkey, + :payment_funding_transactions_increase_ach_strategy_id_fkey, + :payment_funding_transactions_off_platform_strategy_id_fkey, + :payment_funding_transactions_originated_book_transaction_i_fkey, + :payment_funding_transactions_originating_payment_account_i_fkey, + :payment_funding_transactions_platform_ledger_id_fkey, + :payment_funding_transactions_reversal_book_transaction_id_fkey, + :payment_funding_transactions_stripe_card_strategy_id_fkey, + ], + payment_ledgers: [ + :payment_ledgers_account_id_fkey, + ], + payment_payout_transactions: [ + :payment_payout_transactions_crediting_book_transaction_id_fkey, + :payment_payout_transactions_fake_strategy_id_fkey, + :payment_payout_transactions_off_platform_strategy_id_fkey, + :payment_payout_transactions_originated_book_transaction_id_fkey, + :payment_payout_transactions_originating_payment_account_id_fkey, + :payment_payout_transactions_platform_ledger_id_fkey, + :payment_payout_transactions_refunded_funding_transaction_i_fkey, + :payment_payout_transactions_reversal_book_transaction_id_fkey, + :payment_payout_transactions_stripe_charge_refund_strategy__fkey, + ], + payment_triggers: [ + :payment_triggers_originating_ledger_id_fkey, + ], + payment_funding_transaction_increase_ach_strategies: [ + :payment_funding_transaction_in_originating_bank_account_id_fkey, + ], + payment_funding_transaction_stripe_card_strategies: [ + :payment_funding_transaction_stripe_car_originating_card_id_fkey, + ], + payment_trigger_executions: [ + :payment_trigger_executions_book_transaction_id_fkey, + ], + uploaded_files: [ + :uploaded_files_created_by_id_fkey, + ], + }, + } + end + + # Tables which should be entirely truncated. + def tables_to_truncate + return { + public: [ + :message_deliveries, + :member_reset_codes, + :member_sessions, + :member_activities, # these could contain data from members, so clear them out + :organization_registration_links, + :support_notes, + :support_tickets, + + # These are unmodeled legacy and/or auxilliary tables + # which we should trash. + :member_survey_answers, + :member_survey_questions, + :member_surveys, + ], + } + end + + def delete_selective + _delete_legal_entities + _delete_addresses + end + + def _delete_legal_entities + to_keep = Suma::Member.where(roles: Suma::Role.where(name: "admin")).select(:legal_entity_id) + Suma::LegalEntity.exclude(id: to_keep).delete + end + + def _delete_addresses + to_keep = Suma::LegalEntity.dataset.select(:address_id). + union(Suma::Commerce::OfferingFulfillmentOption.dataset.select(:address_id)) + Suma::Address.exclude(id: to_keep).delete + end + + def truncate_analytics + Suma::Analytics::Model.descendants.reject(&:anonymous?).each do |m| + m.dataset.delete + end end end end diff --git a/lib/suma/webhookdb/model.rb b/lib/suma/webhookdb/model.rb index 6ababbef8..fb5ec4ac7 100644 --- a/lib/suma/webhookdb/model.rb +++ b/lib/suma/webhookdb/model.rb @@ -15,6 +15,8 @@ class Suma::Webhookdb::Model extend Suma::Postgres::ModelUtilities class << self + def schema = :public + def db # If models are enabled, assume the configured tables existing in WebhookDB. # If models are not enabled, we can mock out the connection with a mock:// database connection diff --git a/spec/suma/postgres_spec.rb b/spec/suma/postgres_spec.rb index a75126467..9e903a26b 100755 --- a/spec/suma/postgres_spec.rb +++ b/spec/suma/postgres_spec.rb @@ -95,4 +95,16 @@ def self.read_only? = true described_class.run_all_migrations end end + + describe "drop_all_tables" do + it "tries to drop all tables" do + sc = Suma::Postgres::Model + described_class.register_model_superclass(sc) + sc.db.transaction(rollback: :always) do + expect(sc.db.tables.count).to be > 10 + described_class.drop_all_tables + expect(sc.db.tables.count).to be < 10 + end + end + end end diff --git a/spec/tasks/db_spec.rb b/spec/tasks/db_spec.rb index 4d77b914d..9492c6c80 100644 --- a/spec/tasks/db_spec.rb +++ b/spec/tasks/db_spec.rb @@ -8,9 +8,7 @@ describe "drop_tables" do it "drops all tables" do - expect(described_class).to receive(:exec). - with(be_a(Sequel::Database), match(/DROP TABLE [\w.]+ CASCADE/)). - at_least(10).times + expect(Suma::Postgres).to receive(:drop_all_tables) invoke_rake_task("db:drop_tables") end end diff --git a/spec/tasks/release_spec.rb b/spec/tasks/release_spec.rb index 8bb040c39..620c6fb4d 100644 --- a/spec/tasks/release_spec.rb +++ b/spec/tasks/release_spec.rb @@ -41,4 +41,16 @@ expect(m.refresh.authenticate?("abcd1234")).to be(true) end end + + describe "restore_staging_db_from_dump" do + it "runs" do + Suma::Fixtures.order.create + admin = Suma::Fixtures.member.admin.create + expect(Suma::Member.dataset.all).to have_length(2) + expect(described_class::StagingAnonymizer).to receive(:drop_all_tables) + expect(described_class::StagingAnonymizer).to receive(:run_pgrestore).twice + invoke_rake_task("release:restore_staging_db_from_dump", "dump.dump") + expect(Suma::Member.dataset.all).to have_same_ids_as(admin) + end + end end From b992ec6987fece0dcb40a6b422c364b967f813cb Mon Sep 17 00:00:00 2001 From: Rob Galanakis Date: Thu, 25 Jun 2026 17:40:52 -0700 Subject: [PATCH 4/7] Convert some FKs to ON DELETE CASCADE Found while working on the release fixup, these should logically already be CASCADE. --- db/migrations/112_charge_analytics_created.rb | 33 ++++++++++++++++++- lib/suma/tasks/release.rb | 29 ++-------------- 2 files changed, 35 insertions(+), 27 deletions(-) diff --git a/db/migrations/112_charge_analytics_created.rb b/db/migrations/112_charge_analytics_created.rb index fb50c74b5..6d1a15f37 100644 --- a/db/migrations/112_charge_analytics_created.rb +++ b/db/migrations/112_charge_analytics_created.rb @@ -1,9 +1,40 @@ # frozen_string_literal: true Sequel.migration do - change do + # These FK columns should use ON DELETE CASCADE, not the default/restrict + fks_to_cascade = [ + [:charges, :member_id, :members], + [:commerce_carts, :member_id, :members], + [:commerce_cart_items, :cart_id, :commerce_carts], + [:commerce_cart_items, :product_id, :commerce_products], + [:commerce_order_audit_logs, :order_id, :commerce_orders], + [:member_reset_codes, :member_id, :members], + [:message_preferences, :member_id, :members], + [:mobility_trips, :member_id, :members], + [:organization_memberships, :member_id, :members], + [:organization_membership_verifications, :membership_id, :organization_memberships], + ] + up do alter_table(Sequel[:analytics][:charges]) do add_column :incurred_at, :timestamptz end + fks_to_cascade.each do |tbl, col, foreign| + alter_table tbl do + drop_foreign_key [col] + add_foreign_key [col], foreign, on_delete: :cascade + end + end + end + + down do + alter_table(Sequel[:analytics][:charges]) do + drop_column :incurred_at + end + fks_to_cascade.each do |tbl, col, foreign| + alter_table tbl do + drop_foreign_key [col] + add_foreign_key [col], foreign + end + end end end diff --git a/lib/suma/tasks/release.rb b/lib/suma/tasks/release.rb index 55e521c46..e7518fc47 100644 --- a/lib/suma/tasks/release.rb +++ b/lib/suma/tasks/release.rb @@ -209,13 +209,6 @@ def _model_fks anon_proxy_vendor_account_messages: [ :anon_proxy_vendor_account_messages_outbound_delivery_id_fkey, ], - commerce_carts: [ # TODO: migration - :commerce_carts_member_id_fkey, - ], - commerce_cart_items: [ # TODO: migration - :commerce_cart_items_cart_id_fkey, - :commerce_cart_items_product_id_fkey, - ], commerce_checkouts: [ :commerce_checkouts_bank_account_id_fkey, :commerce_checkouts_card_id_fkey, @@ -227,27 +220,9 @@ def _model_fks commerce_orders: [ :commerce_orders_checkout_id_fkey, ], - commerce_order_audit_logs: [ - :commerce_order_audit_logs_order_id_fkey, # TODO: migration - ], marketing_sms_dispatches: [ :marketing_sms_dispatches_member_id_fkey, ], - member_reset_codes: [ - :member_reset_codes_message_delivery_id_fkey, - ], - message_preferences: [ - :message_preferences_member_id_fkey, - ], - mobility_trips: [ - :mobility_trips_member_id_fkey, - ], - organization_memberships: [ - :organization_memberships_member_id_fkey, - ], - organization_membership_verifications: [ - :organization_membership_verifications_membership_id_fkey, # TODO: migrate - ], payment_accounts: [ :payment_accounts_member_id_fkey, ], @@ -311,11 +286,13 @@ def _model_fks def tables_to_truncate return { public: [ + :idempotencies, + :external_credentials, :message_deliveries, :member_reset_codes, :member_sessions, :member_activities, # these could contain data from members, so clear them out - :organization_registration_links, + :organization_registration_links, # would be a security issue :support_notes, :support_tickets, From 8735cdcbd56bdf70edb99fd0d32b705fd98f771e Mon Sep 17 00:00:00 2001 From: Rob Galanakis Date: Thu, 25 Jun 2026 17:50:42 -0700 Subject: [PATCH 5/7] Fix signup agreement div click issue Clicking the div would update the React state, but not the React Hook Form state, which works based on the native input state, which isn't changing with React's rendering. This uses refs to keep things in sync, and drives the state from the checkbox rather than higher up state created through React.useState. --- webapp/src/components/SignupAgreement.jsx | 35 +++++++++++++---------- webapp/src/pages/ContactListAdd.jsx | 8 +----- webapp/src/pages/Start.jsx | 8 +----- 3 files changed, 22 insertions(+), 29 deletions(-) diff --git a/webapp/src/components/SignupAgreement.jsx b/webapp/src/components/SignupAgreement.jsx index 06a2ab1d4..410ecccd8 100644 --- a/webapp/src/components/SignupAgreement.jsx +++ b/webapp/src/components/SignupAgreement.jsx @@ -3,29 +3,34 @@ import FormError from "./FormError.jsx"; import React from "react"; import Form from "react-bootstrap/Form"; -export default function SignupAgreement({ - checked, - errors, - register, - onCheckedChanged, - ...rest -}) { - function handleClick() { - onCheckedChanged(!checked); +export default function SignupAgreement({ errors, register, ...rest }) { + const inputRef = React.useRef(null); + + const { ref: rhfRef, ...registerRest } = register("agree", { + validate: (value) => value === true || t("common.agree_to_continue"), + }); + + function handleDivClick(e) { + // avoid double-toggling if the user clicked the input/label directly + if (e.target === inputRef.current) { + return; + } + inputRef.current?.click(); } + return ( -
+
value === true || t("common.agree_to_continue"), - })} + {...registerRest} + ref={(el) => { + rhfRef(el); + inputRef.current = el; + }} {...rest} - onChange={(e) => onCheckedChanged(e.target.checked)} />
diff --git a/webapp/src/pages/ContactListAdd.jsx b/webapp/src/pages/ContactListAdd.jsx index 513e4cb76..8d2bbac25 100644 --- a/webapp/src/pages/ContactListAdd.jsx +++ b/webapp/src/pages/ContactListAdd.jsx @@ -38,7 +38,6 @@ export default function ContactListAdd() { const [phone, setPhone] = React.useState(""); const [referral, setReferral] = React.useState(""); const [organizationName, setOrganizationName] = React.useState(""); - const [agreementChecked, setAgreementChecked] = React.useState(false); const handleFormSubmit = () => { api .authContactList({ @@ -139,12 +138,7 @@ export default function ContactListAdd() { errors={errors} />
- + - + Date: Thu, 25 Jun 2026 18:04:39 -0700 Subject: [PATCH 6/7] Marketing SMS dispatch: Save to DB on error We were losing error information. --- lib/suma/marketing/sms_dispatch.rb | 4 +++- spec/suma/marketing/sms_dispatch_spec.rb | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/suma/marketing/sms_dispatch.rb b/lib/suma/marketing/sms_dispatch.rb index a9d7512a2..014b113c8 100644 --- a/lib/suma/marketing/sms_dispatch.rb +++ b/lib/suma/marketing/sms_dispatch.rb @@ -94,11 +94,13 @@ def dispatch!(log_tags: {}) self.logger.error("dispatch_marketing_broadcast_error", e) Sentry.capture_exception(e, tags: log_tags) self.last_error = e.to_s + self.save_changes return self end self.logger.info("dispatched_marketing_broadcast", signalwire_message_id: sw_resp.sid) self.set_sent(sw_resp.sid) - return self.save_changes + self.save_changes + return self end def rel_admin_link = "/marketing-sms-dispatch/#{self.id}" diff --git a/spec/suma/marketing/sms_dispatch_spec.rb b/spec/suma/marketing/sms_dispatch_spec.rb index c8d8bd8e0..d30e41576 100644 --- a/spec/suma/marketing/sms_dispatch_spec.rb +++ b/spec/suma/marketing/sms_dispatch_spec.rb @@ -153,6 +153,8 @@ expect(disp.refresh).to have_attributes( sent?: false, transport_message_id: nil, + status: :pending, + last_error: start_with("[HTTP 400] 123"), ) end From 033ccc97be60b6783c1c7ba405cdc83827833ddf Mon Sep 17 00:00:00 2001 From: Rob Galanakis Date: Fri, 26 Jun 2026 08:13:38 -0700 Subject: [PATCH 7/7] Fix flaky test in admin_api_spec.rb --- spec/suma/admin_api_spec.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spec/suma/admin_api_spec.rb b/spec/suma/admin_api_spec.rb index a69fe6ce9..be8a9bb7d 100644 --- a/spec/suma/admin_api_spec.rb +++ b/spec/suma/admin_api_spec.rb @@ -205,7 +205,9 @@ def method_missing(*); end expect(Suma::Vendor).to receive(:method_defined?). with(:products_dataset). and_return(false). - twice + # Not entirely sure why but this can be called once or twice + # Run the specs in this file repeatedly to reproduce (depends on seed). + at_least(:once) ent = Class.new(Suma::AdminAPI::Entities::BaseModelEntity) do model Suma::Vendor expose_related :products, with: Suma::AdminAPI::Entities::BaseModelEntity