Analytics charges; staging restore script; signup agreement checkbox fix; marketing dispatch error save

Makefile

@@ -6,6 +6,8 @@ production_app:=suma-production
 OUT ?= -
 MESSAGE_LANG ?=
 MESSAGE_TRANSPORT ?=
+DBDUMP = temp/latest.dump
+DBURL_LOCAL = postgres://suma:suma@localhost:22005/suma
 
 install:
 	bundle install
@@ -102,7 +104,7 @@ annotate:
 	RACK_ENV=test LOG_LEVEL=info bundle exec rake annotate
 
 psql: cmd-exists-pgcli
-	pgcli postgres://suma:suma@localhost:22005/suma
+	pgcli $(DBURL_LOCAL)
 psql-test: cmd-exists-pgcli
 	pgcli postgres://suma:suma@localhost:22006/suma_test
 psql-%: cmd-exists-pgcli
@@ -134,31 +136,36 @@ analytics-reimport:
 	@bundle exec rake analytics:truncate
 	@bundle exec rake analytics:import
 
+analytics-reimport-production:
+	heroku run:detached bundle exec rake analytics:import --app $(production_app)
+
 take-production-db-snapshot:
 	heroku pg:backups:capture --app $(production_app)
 
 download-production-dump:
 	@mkdir -p temp
-	@rm -f latest.dump
-	heroku pg:backups:download --app $(production_app)
-	@mv latest.dump temp/latest.dump
+	@rm -f $(DBDUMP)
+	heroku pg:backups:download -o $(DBDUMP) --app $(production_app)
+	@./bin/notify "Downloaded production dump"
 
-restore-db-from-dump:
+restore-dump-for-local-db:
 	@bundle exec rake db:drop_tables
-	@mkdir -p temp
-	@PGPASSWORD=suma psql postgres://suma:suma@localhost:22005/suma -c "CREATE SCHEMA IF NOT EXISTS heroku_ext; ALTER DATABASE suma SET search_path TO public,heroku_ext;"
-	PGPASSWORD=suma pg_restore --clean --no-acl --no-owner -h 127.0.0.1 -p 22005 -U suma -d suma temp/latest.dump || true
-	@PGPASSWORD=suma psql postgres://suma:suma@localhost:22005/suma -c "ALTER EXTENSION citext SET SCHEMA public"
-	@bundle exec rake release:prepare_prod_db_for_testing
-	@./bin/notify "Finished restoring database from production"
+	@PGPASSWORD=suma psql $(DBURL_LOCAL) -c "CREATE SCHEMA IF NOT EXISTS heroku_ext; ALTER DATABASE suma SET search_path TO public,heroku_ext;"
+	PGPASSWORD=suma pg_restore --clean --if-exists --no-acl --no-owner -d $(DBURL_LOCAL) $(DBDUMP) || true
+	@PGPASSWORD=suma psql $(DBURL_LOCAL) -c "ALTER EXTENSION citext SET SCHEMA public"
+	@bundle exec rake release:prepare_prod_db_for_local
+	@./bin/notify "Finished restoring database from dump"
 
+restore-dump-for-staging-db:
+	@bundle exec rake release:restore_staging_db_from_dump[$(DBDUMP)]
+	@./bin/notify "Finished restoring staging from dump"
 
 reinit-db-from-dump:
 	docker compose down -v
 	docker compose up -d
 	sleep 5
-	make restore-db-from-dump
-	@echo "Remember to migrate your test DB before running tests by running 'make migrate-test'"
+	make restore-dump-for-local-db
+	make migrate-test
 
 build-webapp:
 	@bundle exec rake frontend:build_webapp

bin/notify

@@ -1,4 +1,16 @@
 #!/usr/bin/env bash
 set -e
 
-osascript -e "display notification \"$1\" with title \"Suma\""
+MESSAGE="$1"
+TITLE="Suma"
+
+case "$(uname -s)" in
+  Darwin)
+    osascript -e "display notification \"$MESSAGE\" with title \"$TITLE\""
+    ;;
+  Linux)
+    if command -v notify-send >/dev/null 2>&1; then
+      notify-send "$TITLE" "$MESSAGE"
+    fi
+    ;;
+esac

db/migrations/112_charge_analytics_created.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  # These FK columns should use ON DELETE CASCADE, not the default/restrict
+  fks_to_cascade = [
+    [:charges, :member_id, :members],
+    [:commerce_carts, :member_id, :members],
+    [:commerce_cart_items, :cart_id, :commerce_carts],
+    [:commerce_cart_items, :product_id, :commerce_products],
+    [:commerce_order_audit_logs, :order_id, :commerce_orders],
+    [:member_reset_codes, :member_id, :members],
+    [:message_preferences, :member_id, :members],
+    [:mobility_trips, :member_id, :members],
+    [:organization_memberships, :member_id, :members],
+    [:organization_membership_verifications, :membership_id, :organization_memberships],
+  ]
+  up do
+    alter_table(Sequel[:analytics][:charges]) do
+      add_column :incurred_at, :timestamptz
+    end
+    fks_to_cascade.each do |tbl, col, foreign|
+      alter_table tbl do
+        drop_foreign_key [col]
+        add_foreign_key [col], foreign, on_delete: :cascade
+      end
+    end
+  end
+
+  down do
+    alter_table(Sequel[:analytics][:charges]) do
+      drop_column :incurred_at
+    end
+    fks_to_cascade.each do |tbl, col, foreign|
+      alter_table tbl do
+        drop_foreign_key [col]
+        add_foreign_key [col], foreign
+      end
+    end
+  end
+end

lib/suma/analytics/charge.rb

@@ -7,19 +7,26 @@ class Suma::Analytics::Charge < Suma::Analytics::Model(Sequel[:analytics][:charg
 
   destroy_from Suma::Charge
 
-  denormalize Suma::Charge, with: [
-    [:charge_id, :id],
-    :opaque_id,
-    :created_at,
-    :member_id,
-    [:order_id, :commerce_order_id],
-    [:trip_id, :mobility_trip_id],
-    :undiscounted_subtotal,
-    :discounted_subtotal,
-    :discount_amount,
-    [:cash_paid, :cash_paid_from_ledger],
-    [:noncash_paid, :noncash_paid_from_ledger],
-  ]
+  denormalize Suma::Charge, with: :denormalize_charge
+
+  def self.denormalize_charge(charge)
+    # one or the other must be set
+    incurred_at = charge.mobility_trip&.ended_at || charge.commerce_order.created_at
+    return {
+      charge_id: charge.id,
+      opaque_id: charge.opaque_id,
+      created_at: charge.created_at,
+      member_id: charge.member_id,
+      order_id: charge.commerce_order_id,
+      trip_id: charge.mobility_trip_id,
+      incurred_at:,
+      undiscounted_subtotal: charge.undiscounted_subtotal,
+      discounted_subtotal: charge.discounted_subtotal,
+      discount_amount: charge.discount_amount,
+      cash_paid: charge.cash_paid_from_ledger,
+      noncash_paid: charge.noncash_paid_from_ledger,
+    }
+  end
 end
 
 # Table: analytics.charges

lib/suma/analytics/model.rb

@@ -36,6 +36,8 @@ class RowMismatch < StandardError; end
     end
   end
 
+  def self.schema = :analytics
+
   def self.inherited(subclass)
     super
     subclass.extend(ClassMethods)
@@ -63,12 +65,12 @@ def unique_key(sym=nil)
     #   - A +Proc+, called with the transactional model instance.
     #   - An +Array+, which is a shorthand for denormalization. Each item in the array is one of:
     #     - A +Symbol+, like `:name`,
-    #       which would add a cell for `name=model.name`.
-    #     - A tuple of symbols, like `[:id, :member_id]`,
-    #       which would add a cell for `member_id=model_id`.
-    #     - A tuple of a symbol and a symbol array, like `[:id, [:member, :id]]`,
-    #       which would add a cell for `member_id=member.id`.
-    #     - A tuple of a symbol and proc, like `[:email, ->(m) { m.email.upcase }]`,
+    #       which would add a cell for `name=tmodel.name`.
+    #     - A tuple of symbols, like `[:member_id, :id]`,
+    #       which would add a cell for `member_id=tmodel.id`.
+    #     - A tuple of a symbol and a symbol array, like `[:member_id, [:member, :id]]`,
+    #       which would add a cell for `member_id=tmodel.member.id`.
+    #     - A tuple of a symbol and proc, like `[:email, ->(tmodel) { tmodel.email.upcase }]`,
     #       called with the model instance, which would add a cell like `email='A@B.C'`.
     def denormalize(transactional_model_class, with:)
       self.denormalizers[transactional_model_class] = with

lib/suma/marketing/sms_dispatch.rb

@@ -94,11 +94,13 @@ def dispatch!(log_tags: {})
       self.logger.error("dispatch_marketing_broadcast_error", e)
       Sentry.capture_exception(e, tags: log_tags)
       self.last_error = e.to_s
+      self.save_changes
       return self
     end
     self.logger.info("dispatched_marketing_broadcast", signalwire_message_id: sw_resp.sid)
     self.set_sent(sw_resp.sid)
-    return self.save_changes
+    self.save_changes
+    return self
   end
 
   def rel_admin_link = "/marketing-sms-dispatch/#{self.id}"

lib/suma/postgres.rb

@@ -235,6 +235,17 @@ def self.load_models
     end
   end
 
+  # Drop all tables in the database for all model superclasses.
+  def self.drop_all_tables
+    self.load_superclasses
+    self.model_superclasses.reject(&:read_only?).each do |sc|
+      schemaname = sc.schema.to_s
+      sc.db[:pg_tables].where(schemaname:).each do |tbl|
+        sc.db.execute("DROP TABLE #{schemaname}.#{tbl[:tablename]} CASCADE")
+      end
+    end
+  end
+
   # Return 'Time.now' as an expression suitable for Sequel/SQL.
   # In some cases (like range @> expressions) you need to cast to a timestamptz explicitly,
   # the implicit cast isn't enough.

lib/suma/postgres/model.rb

@@ -93,6 +93,8 @@ class Suma::Postgres::Model
     end
   end
 
+  def self.schema = :public
+
   def self.extensions
     return [
       "citext",

lib/suma/tasks/db.rb

@@ -11,18 +11,7 @@ def initialize
       desc "Drop all tables in the public schema."
       task :drop_tables do
         require "suma/postgres"
-        Suma::Postgres.load_superclasses
-        # We cannot use load_models to get the schemas they use, in case the models cannot load correctly.
-        # So just hard-code the known schemas that we use.
-        schemas = ["public", "analytics"]
-        Suma::Postgres.model_superclasses.reject(&:read_only?).each do |sc|
-          next if sc == Suma::Webhookdb::Model && Suma::RACK_ENV != "test"
-          schemas.each do |schemaname|
-            sc.db[:pg_tables].where(schemaname:).each do |tbl|
-              Suma::Tasks::DB.exec(sc.db, "DROP TABLE #{schemaname}.#{tbl[:tablename]} CASCADE")
-            end
-          end
-        end
+        Suma::Postgres.drop_all_tables
       end
 
       desc "Remove all data from application schemas"