feat(node): upload local email templates by hash on send

.changeset/node-emails-local-template.md

@@ -0,0 +1,5 @@
+---
+'posthog-node': minor
+---
+
+`posthog.emails.send()` also sends a local template: code content named by a `templateKey`, with `templateData` that PostHog fills in. The SDK uploads the template on its first send and whenever its content changes.

packages/node/references/posthog-node-references-latest.json

@@ -2052,6 +2052,11 @@
           "type": "string",
           "name": "subject"
         },
+        {
+          "description": "Leave it out to send rendered content: the email goes out as is, and PostHog never stores it. Pass it to send a local template: PostHog fills its `{{ variables.* }}`, and the SDK uploads the template when PostHog does not hold its content yet.",
+          "type": "Record<string, unknown>",
+          "name": "templateData"
+        },
         {
           "description": "Names the email, such as `'password-reset'`, so its metrics, logs and status group under it in PostHog.",
           "type": "string",
@@ -4340,7 +4345,7 @@
       "name": "SendCodeEmailOptions",
       "properties": [],
       "path": "src/emails.ts",
-      "example": "SendEmailBase & CodeEmailForm & Without<TemplateEmailForm>"
+      "example": "SendEmailBase & CodeEmailForm & Without<Omit<TemplateEmailForm, 'templateData'>>"
     },
     {
       "id": "SendEmailBase",
@@ -4357,7 +4362,7 @@
           "name": "distinctId"
         },
         {
-          "description": "Sends with a key PostHog already accepted queue nothing and return the first send's `id`, even after the email's workflow was paused or the default sender changed. A deleted template, or a project that no longer has a default sender, still refuses the send. The same key with other options, such as another `captureEngagement` or `replyTo`, sends nothing and returns `invalid_idempotent_request`. Without it, the SDK makes a UUID v4 that covers its own retries. When a job runner retries the send, pass a key derived from what the email is for, such as `welcome:user_42`, and set `maxRetries: 0`.",
+          "description": "Sends with a key PostHog already accepted queue nothing and return the first send's `id`, even after the email's workflow was paused or the default sender changed. A deleted template, a local template whose content changed, or a project that no longer has a default sender while you name no `from`, still refuses the send. The same key with other options, such as another `captureEngagement` or `replyTo`, sends nothing and returns `invalid_idempotent_request`. Without it, the SDK makes a UUID v4 that covers its own retries. When a job runner retries the send, pass a key derived from what the email is for, such as `welcome:user_42`, and set `maxRetries: 0`.",
           "type": "string",
           "name": "idempotencyKey"
         },
@@ -4440,7 +4445,7 @@
           "name": "distinctId"
         },
         {
-          "description": "Sends with a key PostHog already accepted queue nothing and return the first send's `id`, even after the email's\nworkflow was paused or the default sender changed. A deleted template, or a project that no longer has a default\nsender, still refuses the send. The same key with other options, such as another `captureEngagement` or `replyTo`,\nsends nothing and returns `invalid_idempotent_request`. Without it, the SDK makes a UUID v4 that covers its own\nretries. When a job runner retries the send, pass a key derived from what the email is for, such as\n`welcome:user_42`, and set `maxRetries: 0`.",
+          "description": "Sends with a key PostHog already accepted queue nothing and return the first send's `id`, even after the email's\nworkflow was paused or the default sender changed. A deleted template, a local template whose content changed, or\na project that no longer has a default sender while you name no `from`, still refuses the send. The same key with\nother options, such as another `captureEngagement` or `replyTo`, sends nothing and returns\n`invalid_idempotent_request`. Without it, the SDK makes a UUID v4 that covers its own retries. When a job runner\nretries the send, pass a key derived from what the email is for, such as `welcome:user_42`, and set\n`maxRetries: 0`.",
           "type": "string",
           "name": "idempotencyKey"
         },

packages/node/src/__tests__/email-content-hash.spec.ts

@@ -0,0 +1,30 @@
+import { emailContentHash } from '@/email-content-hash'
+
+describe('emailContentHash', () => {
+  it.each([
+    {
+      content: { subject: 'Welcome, {{ variables.first_name }}', html: '<p>Hi {{ variables.first_name }}</p>' },
+      hash: 'v1:f74cd913f3aa2e5667a588f2865634e56693df77793ca594d18e9b5f89e3d2e6',
+    },
+    {
+      content: { subject: 'Grüße', preheader: 'Vorschau', html: '<p>ä</p>', text: 'ä' },
+      hash: 'v1:75cf6fc3189ca26f60da672d29a1d4c89c7db3dd458df3a550a34a4d2a92c149',
+    },
+    {
+      content: {
+        subject: 'Welcome, {{ variables.first_name }}',
+        html: '<p>Hi {{ variables.first_name }}</p>',
+        text: '\r\n\r',
+      },
+      hash: 'v1:8112876ee1b2dd7327f571b04b546ed057bfff5a904d9f0ae507b6a449ab6453',
+    },
+  ])('matches the content_hash v1 test vector of the server: $hash', async ({ content, hash }) => {
+    expect(await emailContentHash(content)).toBe(hash)
+  })
+
+  it('hashes CRLF and lone CR line endings like LF', async () => {
+    const lf = await emailContentHash({ subject: 'Hi', html: '<p>\na\nb\n</p>' })
+
+    expect(await emailContentHash({ subject: 'Hi', html: '<p>\r\na\rb\r\n</p>' })).toBe(lf)
+  })
+})

packages/node/src/__tests__/emails.spec.ts

@@ -9,6 +9,8 @@ const SECRET_KEY = 'phs_secret'
 const APP_HOST = 'https://posthog.example.com'
 const TEMPLATE_ID = '0190f6d2-7c1e-4b3a-9f5e-1c2d3e4f5a6b'
 const SEND = 'POST /api/projects/@current/messaging/send/email/'
+const UPLOAD_WELCOME = 'PUT /api/projects/@current/messaging_templates/by_key/welcome/'
+const WELCOME_HASH = 'v1:e78af62206b6fdc0bd87d56d2a70555a9bb8d4a37617dcbb8a746f86f7db7a63'
 const UUID_V4 = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/
 
 describe('emails', () => {
@@ -155,6 +157,10 @@ describe('emails', () => {
   })
 
   describe('the code form', () => {
+    const welcome = { templateKey: 'welcome', subject: 'Welcome, {{ variables.name }}', html: '<p>Hi</p>' }
+    const sendLocalWelcome = (posthog: PostHog, content: Partial<typeof welcome> = {}) =>
+      posthog.emails.send({ to: JANE, distinctId: 'user_42', ...welcome, ...content, templateData: { name: 'Jane' } })
+
     it('sends inline HTML to the recipient, as the distinct id, with the secret key', async () => {
       const result = await createClient().emails.send({
         to: JANE,
@@ -240,6 +246,84 @@ describe('emails', () => {
         { ...body, to: JANE, identifiers: { distinct_id: 'user_42' } },
       ])
     })
+
+    it('uploads a local template PostHog does not hold, then sends it again with the same idempotency key', async () => {
+      const result = await sendLocalWelcome(createClient({ migratedFrom: 'resend' }))
+
+      expect(result).toEqual({ data: { id: 'message-1', warnings: [] }, error: null })
+      expect(requestLog()).toEqual([SEND, UPLOAD_WELCOME, SEND])
+      const [firstSend, upload, secondSend] = api.received
+      expect(firstSend.body).toEqual({
+        template: { key: 'welcome', hash: WELCOME_HASH },
+        to: JANE,
+        message_data: { name: 'Jane' },
+        identifiers: { distinct_id: 'user_42' },
+        capture_engagement: true,
+      })
+      expect(upload.body).toEqual({
+        name: 'welcome',
+        subject: welcome.subject,
+        html: welcome.html,
+        if_code_managed: true,
+        if_not_superseded: true,
+      })
+      expect(upload.headers).toEqual(expect.objectContaining({ 'x-posthog-migrated-from': 'resend' }))
+      expect(secondSend.body).toEqual(firstSend.body)
+      expect(secondSend.headers['idempotency-key']).toBe(firstSend.headers['idempotency-key'])
+      expect(api.delivered).toEqual([
+        expect.objectContaining({ template: 'welcome', subject: welcome.subject, variables: { name: 'Jane' } }),
+      ])
+    })
+
+    it('sends a local template PostHog holds in one request, and uploads it again once it changes', async () => {
+      const posthog = createClient()
+      await sendLocalWelcome(posthog)
+      api.received.length = 0
+
+      await sendLocalWelcome(posthog)
+      expect(requestLog()).toEqual([SEND])
+
+      await sendLocalWelcome(posthog, { html: '<p>Hi again</p>' })
+      expect(requestLog()).toEqual([SEND, SEND, UPLOAD_WELCOME, SEND])
+      expect(api.localTemplate('welcome')?.html).toBe('<p>Hi again</p>')
+    })
+
+    it('sends a local template with the recipient and data it was called with, even when the caller reuses them', async () => {
+      const posthog = createClient()
+      const options = { to: JANE, ...welcome, templateData: { name: 'Jane' } }
+
+      const toJane = posthog.emails.send(options)
+      options.to = 'bob@example.com'
+      options.templateData.name = 'Bob'
+      const toBob = posthog.emails.send(options)
+      await Promise.all([toJane, toBob])
+
+      expect(api.delivered.map(({ to, variables }) => [to, variables]).sort()).toEqual([
+        ['bob@example.com', { name: 'Bob' }],
+        [JANE, { name: 'Jane' }],
+      ])
+    })
+
+    it('returns the refusal of a key edited in PostHog, and sends nothing', async () => {
+      api.editInPostHog('welcome', { subject: 'Edited', html: '<p>Edited</p>' })
+
+      const result = await sendLocalWelcome(createClient())
+
+      expect(result.error).toEqual(
+        expect.objectContaining({ name: 'validation_error', code: 'template_edited_in_posthog', statusCode: 409 })
+      )
+      expect(requestLog()).toEqual([SEND, UPLOAD_WELCOME])
+      expect(api.delivered).toEqual([])
+    })
+
+    it('returns a second template_outdated instead of uploading again', async () => {
+      api.failWhen(({ method }) => method === 'POST', { status: 409, body: refusalBody('template_outdated', null) })
+
+      const result = await sendLocalWelcome(createClient())
+
+      expect(result.error).toEqual(expect.objectContaining({ code: 'template_outdated' }))
+      expect(requestLog()).toEqual([SEND, UPLOAD_WELCOME, SEND])
+    })
   })
 
   describe('errors', () => {
@@ -383,11 +467,6 @@ describe('emails', () => {
       ['a code send without a subject', { templateKey: 'welcome', html: '<p>Hi</p>' }, 'subject'],
       ['a code send without html or text', { templateKey: 'welcome', subject: 'Hi' }, 'html'],
       ['a code send with html that is not a string', { templateKey: 'welcome', subject: 'Hi', html: 1 }, 'html'],
-      [
-        'a code send with template data, which only the template form takes',
-        { templateKey: 'welcome', subject: 'Hi', html: '<p>Hi</p>', templateData: { name: 'Jane' } },
-        'templateData',
-      ],
       [
         'Resend scheduledAt, which it does not take',
         { templateId: TEMPLATE_ID, scheduledAt: 'in 1 hour' },
@@ -435,23 +514,20 @@ describe('emails', () => {
       expect(requestLog()).toEqual([SEND])
     })
 
-    it.each([
-      [
-        'a sender on the template form',
-        { templateId: TEMPLATE_ID, from: 'Acme <hi@example.com>' },
-        'A template made in PostHog sets its own sender, subject and content.',
-        'Remove from, or send content from your code with templateKey.',
-      ],
-      [
-        'template data on a code send',
-        { templateKey: 'welcome', subject: 'Hi', html: '<p>Hi</p>', templateData: { name: 'Jane' } },
-        'A send with templateKey takes its content from your code.',
-        'Remove templateData, or send a template made in PostHog with templateId.',
-      ],
-    ])('explains why it refuses %s and how to send it', async (_, form, why, fix) => {
-      const result = await createClient().emails.send({ to: JANE, ...form } as unknown as SendEmailOptions)
+    it('explains why it refuses a sender on the template form and how to send it', async () => {
+      const result = await createClient().emails.send({
+        to: JANE,
+        templateId: TEMPLATE_ID,
+        from: 'Acme <hi@example.com>',
+      } as unknown as SendEmailOptions)
 
-      expect(result.error).toEqual(expect.objectContaining({ code: 'invalid_form', why, fix }))
+      expect(result.error).toEqual(
+        expect.objectContaining({
+          code: 'invalid_form',
+          why: 'A template made in PostHog sets its own sender, subject and content.',
+          fix: 'Remove from, or send content from your code with templateKey.',
+        })
+      )
     })
 
     it('returns unexpected_error when the template data cannot be encoded', async () => {

packages/node/src/__tests__/types/emails-forms.ts

@@ -6,6 +6,7 @@ export const forms: SendEmailOptions[] = [
   { to, templateId: '0190f6d2-7c1e-4b3a-9f5e-1c2d3e4f5a6b', templateData: { name: 'Jane' } },
   { to, templateKey: 'reset', subject: 'Reset', html: '<p>Reset</p>', replyTo: ['help@example.com'] },
   { to, templateKey: 'reset', subject: 'Reset', text: 'Reset', from: 'security@example.com', captureEngagement: false },
+  { to, templateKey: 'welcome', subject: 'Hi {{ variables.name }}', html: '<p>Hi</p>', templateData: { name: 'J' } },
 ]
 
 export const wrongCombinations: SendEmailOptions[] = [
@@ -17,6 +18,4 @@ export const wrongCombinations: SendEmailOptions[] = [
   { to, templateKey: 'reset', subject: 'Reset' },
   // @ts-expect-error the code form needs a subject
   { to, templateKey: 'reset', html: '<p>Reset</p>' },
-  // @ts-expect-error rendered content takes no template data
-  { to, templateKey: 'reset', subject: 'Reset', html: '<p>Reset</p>', templateData: { name: 'J' } },
 ]