feat(workflows): cap sandbox sends and refund ses throttles

docs/internal/workflows-email-sending-tiers.md

@@ -70,7 +70,14 @@ Decay, suspension drops, admin recomputes, and the backfill stay silent.
 
 - The batch audience cap is decided when the batch is dispatched. Adding an email step to the workflow while a batch is queued does not re-cap it; the send-time buckets still cap every email at execution. This is why enforcement requires the worker caps to be deployed (see the rollout order).
 - Test-panel sends bypass the team buckets on purpose, matching the per-workflow rate limit.
-- Sandbox sender sends bypass the team buckets without spending their tokens. Sandbox limits run independently. Keep `WORKFLOWS_SANDBOX_SENDER_ENABLED` off until the recipient membership checks, sandbox caps and sandbox pause gate are deployed.
+- Sandbox sender sends bypass the team buckets without spending their tokens. Sandbox limits run independently.
+  The worker holds every sandbox send, test sends included, to a daily cap per project (`WORKFLOWS_SANDBOX_DAILY_TEAM_CAP`, one token per recipient copy) and per recipient address within that organization (`WORKFLOWS_SANDBOX_DAILY_RECIPIENT_CAP`). These are continuously refilling token buckets, not calendar-day counters; an idle bucket starts full and refills its capacity over 24 hours.
+  A reached cap skips the send instead of delaying it. A limiter error or an empty or invalid cap value skips every sandbox send.
+  Cap keys and the final membership check share one organization snapshot, valid for at most one minute. A stalled claim or member lookup that outlives that snapshot skips the send and returns the claim.
+  SES throttle responses return the sandbox claim before the existing delivery retry. Returns are deduplicated, and Valkey ignores a return command that arrives more than five minutes after it was issued.
+  Test sends run inline in the CDP API, so the API reads the same `SES_RATE_LIMITER_VALKEY_*` settings as the email worker. Without them, every sandbox test send is skipped.
+  CDP API opens the SES Valkey pool only when sandbox delivery is enabled; otherwise its MX validation uses the existing local cache and DNS.
+  Deploy membership checks, daily caps, and the shared pause gate first. Configure the SES identity and Valkey pool, enable `WORKFLOWS_SANDBOX_SENDER_ENABLED`, then enable the UI flag.
 - The buckets are token buckets: a full idle bucket plus refill allows up to roughly twice the stated cap in the very first period. The bucket TTLs exceed the refill periods so this does not recur from idling.
 - A denied send parks until every short bucket has refilled enough to cover it, instead of retrying on a fixed few-minute cadence.
   The computed wait is capped at one hour and then jittered 1x to 2x, so a parked send can wait just under two hours between attempts.

nodejs/src/cdp/cdp-services.ts

@@ -170,6 +170,8 @@ export type CdpCoreServicesConfig = Pick<
         | 'SES_TRACKED_CONFIGURATION_SET'
         | 'SES_UNTRACKED_CONFIGURATION_SET'
         | 'WORKFLOWS_SANDBOX_SENDER_ENABLED'
+        | 'WORKFLOWS_SANDBOX_DAILY_TEAM_CAP'
+        | 'WORKFLOWS_SANDBOX_DAILY_RECIPIENT_CAP'
         | 'SES_SANDBOX_TENANT_NAME'
         | 'SES_SANDBOX_CONFIGURATION_SET'
         | 'SES_SANDBOX_FROM_ADDRESS'
@@ -214,13 +216,15 @@ export interface CdpCoreServicesDeps {
     /**
      * SES Valkey pool shared with the SES rate limiter, opened only on pods whose
      * capabilities actually execute email actions (hogflow/email cyclotron workers).
-     * `null` on every other CDP consumer and on cdp-api so idle pods don't hold
-     * open connections against the SES Valkey instance.
+     * Sandbox-enabled cdp-api opens it too, for the sandbox daily caps of test sends.
+     * `null` on every other CDP consumer so idle pods don't hold open connections
+     * against the SES Valkey instance.
      */
     emailValidationValkey: RedisV2 | null
     /**
-     * Dedicated frequency cap Valkey pool, opened only on the same pods as `emailValidationValkey`.
-     * `null` elsewhere, and when no host is configured, which turns the cap off.
+     * Dedicated frequency cap Valkey pool, opened only on the email workers. `null` elsewhere,
+     * cdp-api included because its test sends skip the cap, and when no host is configured,
+     * which turns the cap off.
      */
     frequencyCapValkey: RedisV2 | null
 }
@@ -458,6 +462,9 @@ export function createCdpCoreServices(
     const teamEmailRateLimiter = deps.emailValidationValkey
         ? new RateLimiterService(deps.emailValidationValkey, { name: 'team-email' })
         : null
+    const sandboxEmailRateLimiter = deps.emailValidationValkey
+        ? new RateLimiterService(deps.emailValidationValkey, { name: 'sandbox-email' })
+        : null
     const emailService = new EmailService(
         {
             sesAccessKeyId: config.SES_ACCESS_KEY_ID,
@@ -486,8 +493,11 @@ export function createCdpCoreServices(
                 tenantName: config.SES_SANDBOX_TENANT_NAME,
                 configurationSetName: config.SES_SANDBOX_CONFIGURATION_SET,
                 fromAddress: config.SES_SANDBOX_FROM_ADDRESS,
+                dailyTeamCap: config.WORKFLOWS_SANDBOX_DAILY_TEAM_CAP,
+                dailyRecipientCap: config.WORKFLOWS_SANDBOX_DAILY_RECIPIENT_CAP,
             },
             deps.teamManager,
+            sandboxEmailRateLimiter,
             redis
         ),
         new OrganizationMembersService(deps.postgres)
@@ -556,9 +566,9 @@ export function createCdpCoreServices(
             : undefined
     )
     // MX verdicts live on the dedicated SES Valkey (same instance as the SES rate
-    // limiter, separate pool). The pool is created by the server only on pods
-    // whose capabilities execute email actions; everywhere else this is null
-    // and EmailValidationService degrades to the local cache + DNS.
+    // limiter, separate pool). The server creates the pool only on pods whose
+    // capabilities execute email actions and on sandbox-enabled cdp-api; everywhere
+    // else this is null and EmailValidationService degrades to the local cache + DNS.
     const emailValidationService = new EmailValidationService(deps.emailValidationValkey)
     const cohortMembershipRepository = new PostgresCohortMembershipRepository(deps.postgres)
     // Observer writes to both stores; the read source decides which verdict drives the metric.

nodejs/src/cdp/config.ts

@@ -167,6 +167,9 @@ export type CdpConfig = ClickhouseConfig & {
     // Empty means not provisioned: tracking-off sends fall back to the tracked set with a warning.
     SES_UNTRACKED_CONFIGURATION_SET: string
     WORKFLOWS_SANDBOX_SENDER_ENABLED: boolean
+    // An empty or invalid value blocks every sandbox send.
+    WORKFLOWS_SANDBOX_DAILY_TEAM_CAP: string
+    WORKFLOWS_SANDBOX_DAILY_RECIPIENT_CAP: string
     SES_SANDBOX_TENANT_NAME: string
     SES_SANDBOX_CONFIGURATION_SET: string
     SES_SANDBOX_FROM_ADDRESS: string
@@ -412,6 +415,8 @@ export function getDefaultCdpConfig(): CdpConfig {
         SES_TRACKED_CONFIGURATION_SET: 'posthog-messaging',
         SES_UNTRACKED_CONFIGURATION_SET: '',
         WORKFLOWS_SANDBOX_SENDER_ENABLED: false,
+        WORKFLOWS_SANDBOX_DAILY_TEAM_CAP: '25',
+        WORKFLOWS_SANDBOX_DAILY_RECIPIENT_CAP: '5',
         SES_SANDBOX_TENANT_NAME: '',
         SES_SANDBOX_CONFIGURATION_SET: '',
         SES_SANDBOX_FROM_ADDRESS: '',