diff --git a/nodejs/src/cdp/cdp-api.serial.test.ts b/nodejs/src/cdp/cdp-api.serial.test.ts index 661d619a6588..16af32001b7b 100644 --- a/nodejs/src/cdp/cdp-api.serial.test.ts +++ b/nodejs/src/cdp/cdp-api.serial.test.ts @@ -2273,6 +2273,11 @@ describe('CDP API', () => { { send: { ...codeSend, tags: [{ name: 'plan', value: 'free' }] } }, { send: { ...codeSend, tags: [{ name: 'plan', value: 'pro' }] } }, ], + [ + 'another attachment', + { send: { ...codeSend, attachments: [{ filename: 'invoice.pdf', content: 'Zmlyc3Q=' }] } }, + { send: { ...codeSend, attachments: [{ filename: 'invoice.pdf', content: 'c2Vjb25k' }] } }, + ], ])('rejects the key for %s', async (_desc, first, retry) => { await sendKeyed({ ...keyed, ...first }) @@ -2388,6 +2393,13 @@ describe('CDP API', () => { ['a sender without an integration', { ...codeSend, from: { email: 'security@example.com' } }], ['no tracking choice', { content: codeSend.content }], ['a copy that names two recipients', { ...codeSend, cc: ['jane@example.com, joe@example.com'] }], + [ + 'attachments over the size cap', + { + ...codeSend, + attachments: [{ filename: 'big.pdf', content: Buffer.alloc(512 * 1024 + 1).toString('base64') }], + }, + ], ])('refuses a code send with %s', async (_desc, acceptedSend) => { const invalidBefore = await outcomeCount('invalid_request') diff --git a/nodejs/src/cdp/rerun/rerun-paginator.replay-fidelity.test.ts b/nodejs/src/cdp/rerun/rerun-paginator.replay-fidelity.test.ts index 7e74984676cc..f5fb6d3f26a0 100644 --- a/nodejs/src/cdp/rerun/rerun-paginator.replay-fidelity.test.ts +++ b/nodejs/src/cdp/rerun/rerun-paginator.replay-fidelity.test.ts @@ -6,7 +6,7 @@ import { CyclotronJobQueuePostgresV2 } from '../services/job-queue/job-queue-pos import { JobQueue } from '../services/job-queue/job-queue.interface' import { HogFunctionManagerService } from '../services/managers/hog-function-manager.service' import { HogFunctionMonitoringService } from '../services/monitoring/hog-function-monitoring.service' -import { HogInvocationResultsService } from '../services/monitoring/hog-invocation-results.service' +import { ATTACHMENTS_OMITTED, HogInvocationResultsService } from '../services/monitoring/hog-invocation-results.service' import { CyclotronJobInvocationHogFlow } from '../types' import { RerunJobState } from './rerun-job.types' import { RerunPaginatorService } from './rerun-paginator.service' @@ -145,6 +145,38 @@ describe('RerunPaginatorService replay fidelity (hog_flow)', () => { expect(invocation.state?.rerunAttempts).toBe(1) }) + it('skips a code send whose attachments were not stored and reruns the rest of the page', async () => { + const row = (invocationId: string, persistedState: Record): Record => ({ + invocation_id: invocationId, + parent_run_id: '', + attempts: 0, + last_scheduled_at: '2026-06-01 09:00:00.000000', + first_scheduled_at: '2026-06-01 09:00:00.000000', + invocation_globals: JSON.stringify(persistedState), + }) + const event = { uuid: 'evt-3', distinct_id: 'd-3', properties: {}, timestamp: '2026-06-01T09:00:00Z' } + const rows = [ + row('inv-attachments', { + event, + send: { + tracking: { captureEngagement: true }, + attachments: [{ filename: 'invite.ics', contentType: 'text/calendar', size: 32 }], + }, + [ATTACHMENTS_OMITTED]: true, + }), + row('inv-plain', { event, actionStepCount: 0, variables: {} }), + ] + const hogFlowQueue = { + queueInvocations: jest.fn().mockResolvedValue(undefined), + } as unknown as jest.Mocked + + const paginator = buildPaginator(fakeClickhouse(rows), hogFlowQueue) + await paginator.processPage(teamId, state, { jobId: 'rerun-3', createdAt: DateTime.now() }) + + const enqueued = hogFlowQueue.queueInvocations.mock.calls.flatMap(([invocations]) => invocations) + expect(enqueued.map((invocation) => invocation.id)).toEqual(['inv-plain']) + }) + it('leaves currentAction undefined for a flow that never progressed (fresh start is correct there)', async () => { const persistedState = { event: { uuid: 'evt-2', distinct_id: 'd-2', properties: {}, timestamp: '2026-06-01T09:00:00Z' }, diff --git a/nodejs/src/cdp/rerun/rerun-paginator.service.ts b/nodejs/src/cdp/rerun/rerun-paginator.service.ts index df284b4cf3a7..a5b3dafef383 100644 --- a/nodejs/src/cdp/rerun/rerun-paginator.service.ts +++ b/nodejs/src/cdp/rerun/rerun-paginator.service.ts @@ -13,6 +13,7 @@ import { JobQueue } from '../services/job-queue/job-queue.interface' import { HogFunctionManagerService } from '../services/managers/hog-function-manager.service' import { HogFunctionMonitoringService } from '../services/monitoring/hog-function-monitoring.service' import { + ATTACHMENTS_OMITTED, HogInvocationResultsService, SENSITIVE_VARIABLES_OMITTED, decodeInvocationGlobals, @@ -776,6 +777,14 @@ export class RerunPaginatorService { }) return null } + if (persistedState[ATTACHMENTS_OMITTED]) { + logger.warn('⚠️', 'Skipping rerun of a code send whose attachments were not stored', { + functionId, + teamId, + invocation_id: row.invocation_id, + }) + return null + } const eventForFilter = persistedState.event ?? {} const filterGlobals: HogFunctionFilterGlobals = convertToHogFunctionFilterGlobal({ event: eventForFilter, diff --git a/nodejs/src/cdp/schema/emailFields.ts b/nodejs/src/cdp/schema/emailFields.ts index 25b17dcfe00b..91a8cbccf8e2 100644 --- a/nodejs/src/cdp/schema/emailFields.ts +++ b/nodejs/src/cdp/schema/emailFields.ts @@ -58,3 +58,42 @@ export const EmailTagSchema = z.object({ export type EmailHeader = z.infer export type EmailTag = z.infer + +export const MAX_EMAIL_ATTACHMENTS = 10 +export const MAX_EMAIL_ATTACHMENTS_BYTES = 512 * 1024 + +export const EmailAttachmentSchema = z + .object({ + filename: z + .string() + .min(1) + .max(255) + .regex(/^[^\x00-\x1f\x7f/\\]+$/), + content: z + .string() + .min(1) + .regex(/^[A-Za-z0-9+/]+={0,2}$/), + contentType: z + .string() + .max(78) + .regex(/^[!-~]+\/[!-~]+(;[ -~]*)?$/) + .optional(), + contentId: z + .string() + .max(78) + .regex(/^[!-;=?-~]+$/) + .optional(), + }) + .readonly() + +export const EmailAttachmentsSchema = z + .array(EmailAttachmentSchema) + .max(MAX_EMAIL_ATTACHMENTS) + .refine( + (attachments) => + attachments.reduce((total, { content }) => total + Buffer.byteLength(content, 'base64'), 0) <= + MAX_EMAIL_ATTACHMENTS_BYTES, + `Attachments can hold ${MAX_EMAIL_ATTACHMENTS_BYTES / 1024}KB in total` + ) + +export type EmailAttachment = z.infer diff --git a/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts b/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts index caa9d3efe235..da7d86f6944b 100644 --- a/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts +++ b/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts @@ -16,6 +16,8 @@ import { PostgresUse } from '~/common/utils/db/postgres' import { parseJSON } from '~/common/utils/json-parse' import { logger } from '~/common/utils/logger' import { FetchResponse, fetch, internalFetch } from '~/common/utils/request' +import { UUIDT } from '~/common/utils/utils' +import { waitForExpect } from '~/tests/helpers/expectations' import { createTestTeamFixture, uniqueTestId } from '~/tests/helpers/sql' import { Hub, Team } from '../../../types' @@ -32,6 +34,7 @@ import { TeamWorkflowsConfigService } from '../managers/team-workflows-config.se import { EmailSuppressionService, emailSuppressionConfigFromEnv } from '../messaging/email-suppression.service' import { EmailValidationService } from '../messaging/email-validation.service' import { EmailService } from '../messaging/email.service' +import { MailDevAPI } from '../messaging/helpers/maildev' import { EmailTrackingCodeSigner } from '../messaging/helpers/tracking-code' import { RecipientPreferencesService } from '../messaging/recipient-preferences.service' import { RecipientTokensService } from '../messaging/recipient-tokens.service' @@ -3369,6 +3372,38 @@ describe('Hogflow Executor', () => { expect(metrics.filter((m) => m.metric_name === 'email_sent')).toHaveLength(sent) }) + it("delivers a code send's attachments", async () => { + const subject = `Your invite ${new UUIDT().toString()}` + const invocation = flowInvocation(emailFlow(codeKeyStep())) + invocation.state.send = { + ...codeSend(), + content: { subject, html: '

See you there

', text: 'See you there' }, + attachments: [ + { + filename: 'invite.ics', + content: Buffer.from('BEGIN:VCALENDAR\r\nEND:VCALENDAR\r\n').toString('base64'), + contentType: 'text/calendar; method=REQUEST', + }, + ], + } + + let result = await executor.execute(invocation) + while (!result.finished) { + result = await executor.execute(result.invocation) + } + + expect(result.error).toBeUndefined() + mockFetch.mockImplementation(jest.requireActual('~/common/utils/request').fetch) + const mailDev = new MailDevAPI() + await waitForExpect(async () => + expect((await mailDev.getEmails()).filter((email) => email.subject === subject)).toHaveLength(1) + ) + const [delivered] = (await mailDev.getEmails()).filter((email) => email.subject === subject) + expect( + delivered.attachments.map((a: { fileName: string; contentType: string }) => [a.fileName, a.contentType]) + ).toEqual([['invite.ics', 'text/calendar']]) + }) + it('should complete the full round-trip: hogflow → email queue → email sent → workflow continues', async () => { const invocation = flowInvocation(emailFlow(storedStep())) diff --git a/nodejs/src/cdp/services/hogflows/hogflow-functions.service.ts b/nodejs/src/cdp/services/hogflows/hogflow-functions.service.ts index 663b072ac51a..a88c065f3d69 100644 --- a/nodejs/src/cdp/services/hogflows/hogflow-functions.service.ts +++ b/nodejs/src/cdp/services/hogflows/hogflow-functions.service.ts @@ -161,7 +161,11 @@ export class HogFlowFunctionsService { ...invocation, hogFunction: { ...hogFunction, - metadata: { ...hogFunction.metadata, capture_engagement: send?.tracking.captureEngagement ?? null }, + metadata: { + ...hogFunction.metadata, + capture_engagement: send?.tracking.captureEngagement ?? null, + attachments: send?.attachments ?? null, + }, }, state: invocation.state.currentAction?.hogFunctionState ?? { globals: await this.buildInputsWithGlobals(hogFunction, globalsWithSource, send), diff --git a/nodejs/src/cdp/services/hogflows/transactional-email.ts b/nodejs/src/cdp/services/hogflows/transactional-email.ts index c3bfb948c147..008fb7699556 100644 --- a/nodejs/src/cdp/services/hogflows/transactional-email.ts +++ b/nodejs/src/cdp/services/hogflows/transactional-email.ts @@ -3,6 +3,7 @@ import { z } from 'zod' import { BareEmailAddressSchema, + EmailAttachmentsSchema, EmailHeaderSchema, EmailTagSchema, MAX_EMAIL_COPIES, @@ -45,6 +46,7 @@ export const AcceptedSendSchema = z bcc: z.array(BareEmailAddressSchema).max(MAX_EMAIL_COPIES).readonly().optional(), headers: z.array(EmailHeaderSchema.readonly()).max(MAX_EMAIL_HEADERS).readonly().optional(), tags: z.array(EmailTagSchema.readonly()).max(MAX_EMAIL_TAGS).readonly().optional(), + attachments: EmailAttachmentsSchema.readonly().optional(), tracking: z.object({ captureEngagement: z.boolean() }).readonly(), }) .readonly() diff --git a/nodejs/src/cdp/services/messaging/email.service.test.ts b/nodejs/src/cdp/services/messaging/email.service.test.ts index 151509fcc2a3..b37e8d38657f 100644 --- a/nodejs/src/cdp/services/messaging/email.service.test.ts +++ b/nodejs/src/cdp/services/messaging/email.service.test.ts @@ -1589,6 +1589,69 @@ describe('EmailService', () => { ) }) + it("sends a code send's attachments beside its recipients and PostHog's headers", async () => { + sendEmailSpy.mockResolvedValue({ MessageId: 'test-message-id' }) + const invite = 'BEGIN:VCALENDAR\r\nMETHOD:REQUEST\r\nEND:VCALENDAR\r\n' + invocation.hogFunction.metadata = { + message_category_type: 'transactional', + attachments: [ + { + filename: 'invite.ics', + content: Buffer.from(invite).toString('base64'), + contentType: 'text/calendar; method=REQUEST', + }, + { filename: 'logo.png', content: Buffer.from('png').toString('base64'), contentId: 'logo' }, + ], + } + invocation.queueParameters = createEmailParams({ + html: '

Hi

', + cc: 'jane@example.com', + bcc: 'audit@example.com', + replyTo: 'help@example.com', + headers: [{ name: 'X-Entity-Ref-ID', value: 'order-123' }], + tags: [{ name: 'category', value: 'invite' }], + }) + + const result = await service.executeSendEmail(invocation) + + expect(result.error).toBeUndefined() + const { input } = sendEmailSpy.mock.calls[0][0] as { input: any } + expect(input).toMatchObject({ + FromEmailAddress: '"Test User" ', + Destination: { + ToAddresses: ['"Test User" '], + CcAddresses: ['jane@example.com'], + BccAddresses: ['audit@example.com'], + }, + ReplyToAddresses: ['help@example.com'], + TenantName: `team-${team.id}`, + }) + expect(input.EmailTags.map((t: { Name: string }) => t.Name)).toEqual(['ph_id', 'category']) + expect(input.Content.Simple.Headers.map((h: { Name: string }) => h.Name)).toEqual([ + 'X-Entity-Ref-ID', + 'X-PostHog-Tracking-Code', + 'Auto-Submitted', + ]) + expect(input.Content.Simple.Attachments).toEqual([ + { + FileName: 'invite.ics', + RawContent: Buffer.from(invite), + ContentType: 'text/calendar; method=REQUEST', + ContentId: undefined, + ContentDisposition: 'ATTACHMENT', + ContentTransferEncoding: 'BASE64', + }, + { + FileName: 'logo.png', + RawContent: Buffer.from('png'), + ContentType: undefined, + ContentId: 'logo', + ContentDisposition: 'INLINE', + ContentTransferEncoding: 'BASE64', + }, + ]) + }) + it('attaches the X-PostHog-Tracking-Code header carrying the full signed code', async () => { // The header is the authoritative tracking-code carrier (the EmailTag is the // bounded backwards-compat fallback). It rides on every outbound message, diff --git a/nodejs/src/cdp/services/messaging/email.service.ts b/nodejs/src/cdp/services/messaging/email.service.ts index 23ffa15179a5..c2349c8c24d4 100644 --- a/nodejs/src/cdp/services/messaging/email.service.ts +++ b/nodejs/src/cdp/services/messaging/email.service.ts @@ -1,10 +1,11 @@ -import { MessageHeader, SESv2Client, SendEmailCommand, SendEmailCommandInput } from '@aws-sdk/client-sesv2' +import { Attachment, MessageHeader, SESv2Client, SendEmailCommand, SendEmailCommandInput } from '@aws-sdk/client-sesv2' import { DateTime } from 'luxon' import { SendMailOptions } from 'nodemailer' +import Mail from 'nodemailer/lib/mailer' import { Counter, Histogram } from 'prom-client' import { CyclotronInvocationQueueParametersEmailType } from '~/cdp/schema/cyclotron' -import { EmailTag } from '~/cdp/schema/emailFields' +import { EmailAttachment, EmailAttachmentsSchema, EmailTag } from '~/cdp/schema/emailFields' import { HogFlowEmailSendingRateLimit, HogFlowEmailSendingRateLimitSchema } from '~/cdp/schema/hogflow' import { CyclotronJobInvocationHogFunction, @@ -321,6 +322,27 @@ export function emailTagProperties(tags: readonly EmailTag[]): Record [tag.name, tag.value])) } +// The workflow puts a code send's attachments on the email step's hog function. Checked again here so +// an attachment from any path reaches the provider only in a shape the send route accepts. +function codeSendAttachments(invocation: CyclotronJobInvocationHogFunction): EmailAttachment[] { + return EmailAttachmentsSchema.parse(invocation.hogFunction.metadata?.attachments ?? []) +} + +function toSesAttachment({ filename, content, contentType, contentId }: EmailAttachment): Attachment { + return { + FileName: filename, + RawContent: Buffer.from(content, 'base64'), + ContentType: contentType, + ContentId: contentId, + ContentDisposition: contentId ? 'INLINE' : 'ATTACHMENT', + ContentTransferEncoding: 'BASE64', + } +} + +function toMailAttachment({ filename, content, contentType, contentId }: EmailAttachment): Mail.Attachment { + return { filename, content: Buffer.from(content, 'base64'), contentType, cid: contentId } +} + export function parseAddressList(value?: string): string[] | undefined { if (!value || !value.trim()) { return undefined @@ -994,7 +1016,13 @@ export class EmailService { const ccAddresses = parseAddressList(params.cc) const bccAddresses = parseAddressList(params.bcc) const replyToAddresses = parseAddressList(params.replyTo) + const attachments = codeSendAttachments(result.invocation) + if (attachments.length) { + mailOptions.attachments = attachments.map(toMailAttachment) + mailOptions.disableFileAccess = true + mailOptions.disableUrlAccess = true + } if (ccAddresses) { mailOptions.cc = ccAddresses } @@ -1060,6 +1088,7 @@ export class EmailService { } : {} + const attachments = codeSendAttachments(result.invocation) const sendEmailParams: SendEmailCommandInput = { FromEmailAddress: formatMailbox(from.name, from.email), Destination: { @@ -1075,6 +1104,7 @@ export class EmailService { ...(params.text ? { Text: { Data: params.text, Charset: 'UTF-8' } } : {}), ...htmlBody, }, + ...(attachments.length ? { Attachments: attachments.map(toSesAttachment) } : {}), }, }, ConfigurationSetName: this.resolveConfigurationSetName(trackingEnabled, result.invocation), diff --git a/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.test.ts b/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.test.ts index c5a3f46a3307..fdfc3993b65e 100644 --- a/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.test.ts +++ b/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.test.ts @@ -8,6 +8,7 @@ import { createExampleInvocation } from '../../_tests/fixtures' import { createExampleHogFlowInvocation } from '../../_tests/fixtures-hogflows' import { CdpOutput } from '../../cdp-services' import { + ATTACHMENTS_OMITTED, HogInvocationResultRow, HogInvocationResultsService, decodeInvocationGlobals, @@ -188,6 +189,26 @@ describe('HogInvocationResultsService', () => { expect({ variables: globals.variables, send: globals.send }).toEqual(expected) }) + it("keeps a code send's attachments in a workflow snapshot as a file list without their content", async () => { + const content = Buffer.from('BEGIN:VCALENDAR\r\nEND:VCALENDAR\r\n').toString('base64') + const invocation = createExampleHogFlowInvocation({ id: 'flow-1' } as any) + invocation.state.send = { + tracking: { captureEngagement: true }, + attachments: [{ filename: 'invite.ics', content, contentType: 'text/calendar' }], + } + await expect(service.recordTerminalFailureDurably(invocation)).resolves.toBe(true) + + const globals = (await decodeInvocationGlobals(parseProducedRows(outputs)[0].invocation_globals)) as Record< + string, + any + > + expect(globals.send.attachments).toEqual([ + { filename: 'invite.ics', contentType: 'text/calendar', size: 32 }, + ]) + expect(globals[ATTACHMENTS_OMITTED]).toBe(true) + expect(JSON.stringify(globals)).not.toContain(content) + }) + it('strips groups and person from invocation_globals — rebuilt downstream on rerun', async () => { const invocation = createExampleInvocation({}, { groups: { diff --git a/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.ts b/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.ts index b92501048907..13c4abe851ec 100644 --- a/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.ts +++ b/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.ts @@ -209,6 +209,7 @@ const INPUT_KEYS: ReadonlySet = new Set(['inputs']) // snapshot keeps no variables or accepted send either, and a rerun can't rebuild the send. const SENSITIVE_KEYS: ReadonlySet = new Set(['inputs', 'variables', 'send']) export const SENSITIVE_VARIABLES_OMITTED = 'sensitive_variables_omitted' +export const ATTACHMENTS_OMITTED = 'attachments_omitted' const stripKeys = (value: T, keys: ReadonlySet): T => { if (value === null || typeof value !== 'object') { @@ -249,6 +250,20 @@ const sendsSensitiveEmail = (invocation: CyclotronJobInvocationHogFlow): boolean return hasSensitiveEmailStep(invocation.hogFlow) } +// A code send's attachments can hold the send route's whole size cap, so a snapshot lists each file +// without its content, and marks that a rerun can't rebuild the send. +const describeAttachments = (state: CyclotronJobInvocationHogFlow['state']): object => { + const send = state?.send + if (!send?.attachments) { + return state ?? {} + } + const attachments = send.attachments.map(({ content, ...file }) => ({ + ...file, + size: Buffer.byteLength(content, 'base64'), + })) + return { ...state, send: { ...send, attachments }, [ATTACHMENTS_OMITTED]: true } +} + const serializeInvocationGlobals = (invocation: CyclotronJobInvocation): string => { if (isHogFunctionInvocation(invocation)) { const { groups: _groups, person: _person, ...globals } = invocation.state.globals @@ -257,13 +272,14 @@ const serializeInvocationGlobals = (invocation: CyclotronJobInvocation): string if (isHogFlowInvocation(invocation)) { // Hog flow state can carry a per-action `currentAction.hogFunctionState.globals.inputs` // — `stripKeys` walks the tree and removes those too. + const state = describeAttachments(invocation.state) if (sendsSensitiveEmail(invocation)) { return JSON.stringify({ - ...stripKeys(invocation.state ?? {}, SENSITIVE_KEYS), + ...stripKeys(state, SENSITIVE_KEYS), [SENSITIVE_VARIABLES_OMITTED]: true, }) } - return JSON.stringify(stripKeys(invocation.state ?? {}, INPUT_KEYS)) + return JSON.stringify(stripKeys(state, INPUT_KEYS)) } return '{}' } diff --git a/products/messaging/backend/facade/contracts.py b/products/messaging/backend/facade/contracts.py index 2d01d247340f..8ce4c43ee0d9 100644 --- a/products/messaging/backend/facade/contracts.py +++ b/products/messaging/backend/facade/contracts.py @@ -100,6 +100,17 @@ class EmailField: value: str +@frozen +class EmailAttachment: + """A file a code send attaches to its email.""" + + filename: str + content: bytes + content_type: str | None = None + # Shows the file inline, where the HTML references it as `cid:`. + content_id: str | None = None + + @frozen class AcceptedSend: """What a code send sets on its own, over what its workflow's email step holds.""" @@ -112,6 +123,7 @@ class AcceptedSend: bcc: tuple[str, ...] = () headers: tuple[EmailField, ...] = () tags: tuple[EmailField, ...] = () + attachments: tuple[EmailAttachment, ...] = () class SendRoute(LabeledStrEnum): diff --git a/products/messaging/backend/facade/transactional_email.py b/products/messaging/backend/facade/transactional_email.py index d64f2cfdcd53..27bc5cb2d394 100644 --- a/products/messaging/backend/facade/transactional_email.py +++ b/products/messaging/backend/facade/transactional_email.py @@ -2,6 +2,7 @@ from products.messaging.backend.facade.contracts import ( AcceptedSend, + EmailAttachment, EmailField, EmailSendCommand, EmailSendOutcome, @@ -32,6 +33,7 @@ "SENDER_ADDRESS_MAX_LENGTH", "SENDER_ADDRESS_PATTERN", "AcceptedSend", + "EmailAttachment", "EmailField", "EmailSendCommand", "EmailSendOutcome", diff --git a/products/messaging/backend/presentation/views/messaging_send.py b/products/messaging/backend/presentation/views/messaging_send.py index 332a38d53266..4ec951419278 100644 --- a/products/messaging/backend/presentation/views/messaging_send.py +++ b/products/messaging/backend/presentation/views/messaging_send.py @@ -22,6 +22,7 @@ SENDER_ADDRESS_MAX_LENGTH, SENDER_ADDRESS_PATTERN, AcceptedSend, + EmailAttachment, EmailField, EmailSendCommand, RenderedContent, @@ -33,6 +34,11 @@ rendered_content, require_code_email_access, ) +from products.messaging.backend.presentation.views.send_email_attachments import ( + MAX_ATTACHMENTS, + MAX_ATTACHMENTS_BYTES, + attachments_field, +) from products.messaging.backend.presentation.views.send_email_fields import ( MAX_COPIES, MAX_HEADER_VALUE_LENGTH, @@ -85,7 +91,7 @@ def validate(self, data: dict[str, Any]) -> dict[str, Any]: CODE_FORM_FIELDS = ("subject", "html", "text") -CODE_SEND_FIELDS = ("from", "reply_to", "cc", "bcc", "headers", "tags") +CODE_SEND_FIELDS = ("from", "reply_to", "cc", "bcc", "headers", "tags", "attachments") class SendIdentifiersSerializer(RefusesUnknownFieldsSerializer, TransactionalEmailIdentifiersSerializer): @@ -162,6 +168,7 @@ class SendEmailSerializer(RefusesUnknownFieldsSerializer, SendTransactionalEmail help_text=f"Name and value pairs, at most {MAX_TAGS}, that PostHog adds to the email's engagement events " "as `$email_tags`. Not with `template.id`.", ) + attachments = attachments_field() capture_engagement = serializers.BooleanField( required=False, help_text="Whether the email's engagement events (sent, delivered, opened, clicked, bounced) are captured " @@ -283,6 +290,9 @@ def template_reference(template: dict[str, Any]) -> TemplateReference: "other standard headers itself, so you can't set them here.", "tags": f"`tags` holds up to {MAX_TAGS} name and value pairs of letters, numbers, `_` and `-`, which PostHog " "adds to the email's engagement events.", + "attachments": f"`attachments` holds up to {MAX_ATTACHMENTS} files of {MAX_ATTACHMENTS_BYTES // 1024} KB in total, " + "each with a plain file name and base64 content. Mailbox providers block some file types, such as `.exe`, so " + "PostHog refuses them.", "capture_engagement": "`capture_engagement` turns the email's engagement events on or off, so it is true or false.", } @@ -499,6 +509,7 @@ def _accepted_send(self, data: dict[str, Any], *, content: RenderedContent | Non EmailField(name=tag["name"], value=tag["value"]) for tag in sorted(data.get("tags", ()), key=lambda tag: tag["name"]) ), + attachments=tuple(EmailAttachment(**attachment) for attachment in data.get("attachments", ())), ) @staticmethod diff --git a/products/messaging/backend/presentation/views/send_email_attachments.py b/products/messaging/backend/presentation/views/send_email_attachments.py new file mode 100644 index 000000000000..36192f621083 --- /dev/null +++ b/products/messaging/backend/presentation/views/send_email_attachments.py @@ -0,0 +1,95 @@ +"""Files a code send attaches to its email, held to what fits in the email queue and what mailbox providers accept.""" + +import re +import base64 +import binascii +from typing import Any + +from rest_framework import serializers + +from products.messaging.backend.presentation.views.send_email_fields import RefusesUnknownFieldsSerializer + +MAX_ATTACHMENTS = 10 +MAX_ATTACHMENTS_BYTES = 512 * 1024 +MAX_FILENAME_LENGTH = 255 +MAX_CONTENT_TYPE_LENGTH = 78 +MAX_CONTENT_ID_LENGTH = 78 + +CONTENT_TYPE = re.compile(r"[!-~]+/[!-~]+(;[ -~]*)?") +CONTENT_ID = re.compile(r"[!-;=?-~]+") +FILENAME_FORBIDDEN = re.compile(r"[\x00-\x1f\x7f/\\]") + +# The file types the email provider refuses to send: https://docs.aws.amazon.com/ses/latest/dg/attachments.html +BLOCKED_EXTENSIONS = frozenset( + """ + ade adp app asp bas bat cer chm cmd com cpl crt csh der exe fxp gadget hlp hta inf ins isp its js jse ksh lib lnk + mad maf mag mam maq mar mas mat mau mav maw mda mdb mde mdt mdw mdz msc msh msh1 msh2 mshxml msh1xml msh2xml msi + msp mst ops pcd pif plg prf prg ps1 ps1xml ps2 ps2xml psc1 psc2 reg scf scr sct shb shs sys tmp url vb vbe vbs vps + vsmacros vss vst vsw vxd ws wsc wsf wsh xnk + """.split() +) + + +class EmailAttachmentSerializer(RefusesUnknownFieldsSerializer): + filename = serializers.CharField( + max_length=MAX_FILENAME_LENGTH, + help_text="File name the recipient sees, such as `invite.ics`. No paths, and no file types mailbox " + "providers block, such as `.exe`.", + ) + content = serializers.CharField(help_text="The file content, Base64 encoded.") + content_type = serializers.CharField( + required=False, + max_length=MAX_CONTENT_TYPE_LENGTH, + help_text="MIME type with its parameters, such as `text/calendar; method=REQUEST`. Without it, the type " + "follows the file name.", + ) + content_id = serializers.CharField( + required=False, + max_length=MAX_CONTENT_ID_LENGTH, + help_text='Shows the file inline: reference it in `html` as `` with `content_id` `logo`.', + ) + + def validate_content(self, value: str) -> bytes: + try: + return base64.b64decode(value, validate=True) + except (binascii.Error, ValueError): + raise serializers.ValidationError("Send the file content as Base64.") from None + + def validate_filename(self, value: str) -> str: + if FILENAME_FORBIDDEN.search(value): + raise serializers.ValidationError("Use a file name without paths or line breaks, such as `invite.ics`.") + name = value.rstrip(". ") + extension = name.rpartition(".")[2].lower() if "." in name else "" + if extension in BLOCKED_EXTENSIONS: + raise serializers.ValidationError( + f"Mailbox providers block `.{extension}` files. Send a link to the file instead." + ) + return value + + def validate_content_type(self, value: str) -> str: + if not CONTENT_TYPE.fullmatch(value): + raise serializers.ValidationError("Use a MIME type, such as `application/pdf`, on one line.") + return value + + def validate_content_id(self, value: str) -> str: + if not CONTENT_ID.fullmatch(value): + raise serializers.ValidationError("Use visible ASCII characters without `<` or `>`, such as `logo`.") + return value + + +def validate_attachments(attachments: list[dict[str, Any]]) -> None: + if sum(len(attachment["content"]) for attachment in attachments) > MAX_ATTACHMENTS_BYTES: + raise serializers.ValidationError( + f"Attachments can hold {MAX_ATTACHMENTS_BYTES // 1024}KB in total. Send a link to larger files." + ) + + +def attachments_field() -> serializers.ListField: + return serializers.ListField( + child=EmailAttachmentSerializer(), + required=False, + max_length=MAX_ATTACHMENTS, + validators=[validate_attachments], + help_text=f"Files to attach, at most {MAX_ATTACHMENTS} and {MAX_ATTACHMENTS_BYTES // 1024}KB in total after " + "decoding. Not with `template.id`.", + ) diff --git a/products/messaging/backend/services/transactional_email.py b/products/messaging/backend/services/transactional_email.py index c6689d040457..3d79a2829127 100644 --- a/products/messaging/backend/services/transactional_email.py +++ b/products/messaging/backend/services/transactional_email.py @@ -2,6 +2,7 @@ per send, so sends get the workflow's metrics, run history and versions.""" import re +import base64 from collections.abc import Callable from email.utils import parseaddr from typing import Any @@ -16,6 +17,7 @@ from products.messaging.backend.facade.contracts import ( AcceptedSend, + EmailAttachment, EmailSendCommand, EmailSendOutcome, MarketingTemplate, @@ -453,8 +455,21 @@ def _send_payload(send: AcceptedSend) -> dict[str, Any]: payload["headers"] = [{"name": header.name, "value": header.value} for header in send.headers] if send.tags: payload["tags"] = [{"name": tag.name, "value": tag.value} for tag in send.tags] + if send.attachments: + payload["attachments"] = [_attachment_payload(attachment) for attachment in send.attachments] return payload +def _attachment_payload(attachment: EmailAttachment) -> dict[str, Any]: + return _without_empty( + { + "filename": attachment.filename, + "content": base64.b64encode(attachment.content).decode(), + "contentType": attachment.content_type, + "contentId": attachment.content_id, + } + ) + + def _without_empty(fields: dict[str, Any]) -> dict[str, Any]: return {name: value for name, value in fields.items() if value} diff --git a/products/messaging/backend/tests/api/test_send_email_fields.py b/products/messaging/backend/tests/api/test_send_email_fields.py index 055f5ae387e7..dfbc3f665a0e 100644 --- a/products/messaging/backend/tests/api/test_send_email_fields.py +++ b/products/messaging/backend/tests/api/test_send_email_fields.py @@ -1,3 +1,4 @@ +import base64 from typing import Any from django.test import SimpleTestCase @@ -90,3 +91,79 @@ def test_refuses_code_send_fields_on_a_template_made_in_posthog(self, field: str assert not body.is_valid() assert field in body.errors, body.errors + + +INVITE = base64.b64encode(b"BEGIN:VCALENDAR\r\nEND:VCALENDAR\r\n").decode() +LONGEST_CONTENT_TYPE = "text/plain; name=" + "a" * 61 + + +def attachment(**fields: Any) -> dict[str, Any]: + return {"filename": "invite.ics", "content": INVITE, **fields} + + +class TestSendEmailAttachments(SimpleTestCase): + def test_accepts_attachments_and_decodes_their_content(self) -> None: + body = SendEmailSerializer( + data=rendered( + attachments=[ + attachment(content_type="text/calendar; method=REQUEST"), + attachment(filename="logo.png", content="iVBORw0KGgo=", content_id="logo"), + attachment(filename="notes.txt", content_type=LONGEST_CONTENT_TYPE, content_id="a" * 78), + ] + ) + ) + + assert body.is_valid(), body.errors + assert body.validated_data["attachments"] == [ + { + "filename": "invite.ics", + "content": b"BEGIN:VCALENDAR\r\nEND:VCALENDAR\r\n", + "content_type": "text/calendar; method=REQUEST", + }, + {"filename": "logo.png", "content": b"\x89PNG\r\n\x1a\n", "content_id": "logo"}, + { + "filename": "notes.txt", + "content": b"BEGIN:VCALENDAR\r\nEND:VCALENDAR\r\n", + "content_type": LONGEST_CONTENT_TYPE, + "content_id": "a" * 78, + }, + ] + + @parameterized.expand( + [ + ("not_base64", [attachment(content="not base64!")]), + ("empty", [attachment(content="")]), + ("over_the_total_size", [attachment(content=base64.b64encode(b"a" * 300_000).decode())] * 2), + ("over_the_count", [attachment(filename=f"file{i}.txt") for i in range(11)]), + ("blocked_extension", [attachment(filename="setup.EXE")]), + ("blocked_extension_with_a_trailing_dot", [attachment(filename="setup.exe. ")]), + ("blocked_extension_alone", [attachment(filename=".exe")]), + ("filename_with_a_line_break", [attachment(filename="invite\r\n.ics")]), + ("filename_with_a_path", [attachment(filename="../invite.ics")]), + ("filename_too_long", [attachment(filename="a" * 252 + ".ics")]), + ("content_type_with_a_line_break", [attachment(content_type="text/plain\r\nBcc: a@example.com")]), + ("content_type_not_a_type", [attachment(content_type="calendar")]), + ("content_id_with_brackets", [attachment(content_id="")]), + ("content_type_too_long", [attachment(content_type=LONGEST_CONTENT_TYPE + "a")]), + ("content_id_too_long", [attachment(content_id="a" * 79)]), + ("unknown_key", [attachment(contentType="text/calendar")]), + ] + ) + def test_refuses_an_attachment_it_cannot_send(self, _name: str, attachments: list[dict[str, Any]]) -> None: + body = SendEmailSerializer(data=rendered(attachments=attachments)) + + assert not body.is_valid() + assert "attachments" in body.errors, body.errors + assert INVITE not in str(body.errors) + + def test_refuses_attachments_on_a_template_made_in_posthog(self) -> None: + body = SendEmailSerializer( + data={ + "to": "user@example.com", + "template": {"id": "4c1f0c3c-8d1b-4a51-9f3a-0d8f1c0f6d2e"}, + "attachments": [attachment()], + } + ) + + assert not body.is_valid() + assert "attachments" in body.errors, body.errors diff --git a/products/messaging/backend/tests/api/test_transactional_email.py b/products/messaging/backend/tests/api/test_transactional_email.py index 85b3b16a2279..31b97b998e37 100644 --- a/products/messaging/backend/tests/api/test_transactional_email.py +++ b/products/messaging/backend/tests/api/test_transactional_email.py @@ -680,6 +680,34 @@ def test_sends_rendered_content_as_is_under_the_keys_workflow(self, mock_post, _ "tracking": {"captureEngagement": True}, }, ), + ( + "rendered_content_with_attachments", + "template_key", + { + "text": "Hi", + "attachments": [ + { + "filename": "invite.ics", + "content": "QkVHSU46VkNBTEVOREFS", + "content_type": "text/calendar; method=REQUEST", + }, + {"filename": "logo.png", "content": "iVBORw0KGgo=", "content_id": "logo"}, + ], + }, + { + "content": {"subject": "Welcome, Jane", "html": "

Hi {{ variables.name }}

", "text": "Hi"}, + "from": {"integrationId": "default", "isDefault": True}, + "attachments": [ + { + "filename": "invite.ics", + "content": "QkVHSU46VkNBTEVOREFS", + "contentType": "text/calendar; method=REQUEST", + }, + {"filename": "logo.png", "content": "iVBORw0KGgo=", "contentId": "logo"}, + ], + "tracking": {"captureEngagement": True}, + }, + ), ( "template_by_key_with_a_copy", "key", @@ -759,6 +787,12 @@ def test_a_send_posts_the_same_headers_and_tags_whatever_their_order(self, mock_ ), ("header_posthog_sets", {"headers": {"From": "a@example.com"}}, "invalid_request", "headers"), ("bad_tag", {"tags": [{"name": "plan", "value": "pro.v2"}]}, "invalid_request", "tags__0__value"), + ( + "blocked_attachment", + {"attachments": [{"filename": "setup.exe", "content": "aGk="}]}, + "invalid_request", + "attachments__0__filename", + ), ] ) def test_refuses_a_rendered_send_it_cannot_make_with_what_to_fix( diff --git a/products/messaging/frontend/generated/api.schemas.ts b/products/messaging/frontend/generated/api.schemas.ts index 8f9ccc65159d..6bee57bfbd9c 100644 --- a/products/messaging/frontend/generated/api.schemas.ts +++ b/products/messaging/frontend/generated/api.schemas.ts @@ -48,6 +48,26 @@ export interface EmailTagApi { value: string } +export interface EmailAttachmentApi { + /** + * File name the recipient sees, such as `invite.ics`. No paths, and no file types mailbox providers block, such as `.exe`. + * @maxLength 255 + */ + filename: string + /** The file content, Base64 encoded. */ + content: string + /** + * MIME type with its parameters, such as `text/calendar; method=REQUEST`. Without it, the type follows the file name. + * @maxLength 78 + */ + content_type?: string + /** + * Shows the file inline: reference it in `html` as `` with `content_id` `logo`. + * @maxLength 78 + */ + content_id?: string +} + export interface SendEmailApi { /** Email address of the recipient, as `jane@example.com`. */ to: string @@ -87,6 +107,11 @@ export interface SendEmailApi { * @maxItems 10 */ tags?: EmailTagApi[] + /** + * Files to attach, at most 10 and 512KB in total after decoding. Not with `template.id`. + * @maxItems 10 + */ + attachments?: EmailAttachmentApi[] /** Whether the email's engagement events (sent, delivered, opened, clicked, bounced) are captured as analytics events of the recipient. They count as analytics events. Defaults to on for `template_key` and `template.key` sends, and to the project's setting for `template.id` sends. */ capture_engagement?: boolean /** Sender, as `security@example.com` or `Example `, on a verified domain of the project. Without it, rendered content goes out from the project's default sender, and a template from its own sender. Not with `template.id`. */ diff --git a/products/messaging/frontend/generated/api.zod.ts b/products/messaging/frontend/generated/api.zod.ts index 14803b117560..ceafc7d1c605 100644 --- a/products/messaging/frontend/generated/api.zod.ts +++ b/products/messaging/frontend/generated/api.zod.ts @@ -27,6 +27,14 @@ export const messagingSendEmailCreateBodyTagsItemValueMax = 256 export const messagingSendEmailCreateBodyTagsMax = 10 +export const messagingSendEmailCreateBodyAttachmentsItemFilenameMax = 255 + +export const messagingSendEmailCreateBodyAttachmentsItemContentTypeMax = 78 + +export const messagingSendEmailCreateBodyAttachmentsItemContentIdMax = 78 + +export const messagingSendEmailCreateBodyAttachmentsMax = 10 + export const MessagingSendEmailCreateBody = /* @__PURE__ */ zod.object({ to: zod.email().describe('Email address of the recipient, as `jane@example.com`.'), identifiers: zod @@ -129,6 +137,35 @@ export const MessagingSendEmailCreateBody = /* @__PURE__ */ zod.object({ .describe( "Name and value pairs, at most 10, that PostHog adds to the email's engagement events as `$email_tags`. Not with `template.id`." ), + attachments: zod + .array( + zod.object({ + filename: zod + .string() + .max(messagingSendEmailCreateBodyAttachmentsItemFilenameMax) + .describe( + 'File name the recipient sees, such as `invite.ics`. No paths, and no file types mailbox providers block, such as `.exe`.' + ), + content: zod.string().describe('The file content, Base64 encoded.'), + content_type: zod + .string() + .max(messagingSendEmailCreateBodyAttachmentsItemContentTypeMax) + .optional() + .describe( + 'MIME type with its parameters, such as `text\/calendar; method=REQUEST`. Without it, the type follows the file name.' + ), + content_id: zod + .string() + .max(messagingSendEmailCreateBodyAttachmentsItemContentIdMax) + .optional() + .describe( + 'Shows the file inline: reference it in `html` as `` with `content_id` `logo`.' + ), + }) + ) + .max(messagingSendEmailCreateBodyAttachmentsMax) + .optional() + .describe('Files to attach, at most 10 and 512KB in total after decoding. Not with `template.id`.'), capture_engagement: zod .boolean() .optional() diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts index 014e03963367..fc7249a21c77 100644 --- a/services/mcp/src/api/generated.ts +++ b/services/mcp/src/api/generated.ts @@ -39872,6 +39872,26 @@ export namespace Schemas { name: string; } + export interface EmailAttachment { + /** + * File name the recipient sees, such as `invite.ics`. No paths, and no file types mailbox providers block, such as `.exe`. + * @maxLength 255 + */ + filename: string; + /** The file content, Base64 encoded. */ + content: string; + /** + * MIME type with its parameters, such as `text/calendar; method=REQUEST`. Without it, the type follows the file name. + * @maxLength 78 + */ + content_type?: string; + /** + * Shows the file inline: reference it in `html` as `` with `content_id` `logo`. + * @maxLength 78 + */ + content_id?: string; + } + /** * * `ai` - AI * * `template` - Template @@ -105892,6 +105912,11 @@ export namespace Schemas { * @maxItems 10 */ tags?: EmailTag[]; + /** + * Files to attach, at most 10 and 512KB in total after decoding. Not with `template.id`. + * @maxItems 10 + */ + attachments?: EmailAttachment[]; /** Whether the email's engagement events (sent, delivered, opened, clicked, bounced) are captured as analytics events of the recipient. They count as analytics events. Defaults to on for `template_key` and `template.key` sends, and to the project's setting for `template.id` sends. */ capture_engagement?: boolean; /** Sender, as `security@example.com` or `Example `, on a verified domain of the project. Without it, rendered content goes out from the project's default sender, and a template from its own sender. Not with `template.id`. */ diff --git a/services/mcp/src/generated/email_templates/api.ts b/services/mcp/src/generated/email_templates/api.ts index c08aa7755153..ab1a8008f4c1 100644 --- a/services/mcp/src/generated/email_templates/api.ts +++ b/services/mcp/src/generated/email_templates/api.ts @@ -43,6 +43,14 @@ export const messagingSendEmailCreateBodyTagsItemValueMax = 256 export const messagingSendEmailCreateBodyTagsMax = 10 +export const messagingSendEmailCreateBodyAttachmentsItemFilenameMax = 255 + +export const messagingSendEmailCreateBodyAttachmentsItemContentTypeMax = 78 + +export const messagingSendEmailCreateBodyAttachmentsItemContentIdMax = 78 + +export const messagingSendEmailCreateBodyAttachmentsMax = 10 + export const MessagingSendEmailCreateBody = () => zod.object({ to: zod.email().describe('Email address of the recipient, as `jane@example.com`.'), identifiers: zod @@ -145,6 +153,35 @@ export const MessagingSendEmailCreateBody = () => zod.object({ .describe( "Name and value pairs, at most 10, that PostHog adds to the email's engagement events as `$email_tags`. Not with `template.id`." ), + attachments: zod + .array( + zod.object({ + filename: zod + .string() + .max(messagingSendEmailCreateBodyAttachmentsItemFilenameMax) + .describe( + 'File name the recipient sees, such as `invite.ics`. No paths, and no file types mailbox providers block, such as `.exe`.' + ), + content: zod.string().describe('The file content, Base64 encoded.'), + content_type: zod + .string() + .max(messagingSendEmailCreateBodyAttachmentsItemContentTypeMax) + .optional() + .describe( + 'MIME type with its parameters, such as `text\/calendar; method=REQUEST`. Without it, the type follows the file name.' + ), + content_id: zod + .string() + .max(messagingSendEmailCreateBodyAttachmentsItemContentIdMax) + .optional() + .describe( + 'Shows the file inline: reference it in `html` as `` with `content_id` `logo`.' + ), + }) + ) + .max(messagingSendEmailCreateBodyAttachmentsMax) + .optional() + .describe('Files to attach, at most 10 and 512KB in total after decoding. Not with `template.id`.'), capture_engagement: zod .boolean() .optional() diff --git a/services/mcp/src/tools/generated/email_templates.ts b/services/mcp/src/tools/generated/email_templates.ts index 6d9d4a48b0d3..f1409da88eef 100644 --- a/services/mcp/src/tools/generated/email_templates.ts +++ b/services/mcp/src/tools/generated/email_templates.ts @@ -361,6 +361,9 @@ const workflowsSendEmail = (): ToolBase` with `content_id` `logo`.", + "maxLength": 78, + "type": "string" + }, + "content_type": { + "description": "MIME type with its parameters, such as `text/calendar; method=REQUEST`. Without it, the type follows the file name.", + "maxLength": 78, + "type": "string" + }, + "filename": { + "description": "File name the recipient sees, such as `invite.ics`. No paths, and no file types mailbox providers block, such as `.exe`.", + "maxLength": 255, + "type": "string" + } + }, + "required": ["filename", "content"], + "type": "object" + }, + "maxItems": 10, + "type": "array" + }, "bcc": { "description": "Addresses that get a hidden copy, at most 10. Each one is checked against the suppression list, and one suppressed address stops the send. Not with `template.id`.", "items": {