diff --git a/server/services/manualReviewToolService/manualReviewToolService.test.ts b/server/services/manualReviewToolService/manualReviewToolService.test.ts index 289b6bdf..4987de8b 100644 --- a/server/services/manualReviewToolService/manualReviewToolService.test.ts +++ b/server/services/manualReviewToolService/manualReviewToolService.test.ts @@ -1,4 +1,3 @@ -/* eslint-disable max-lines */ import { uid } from 'uid'; import { v1 as uuidv1 } from 'uuid'; diff --git a/server/services/manualReviewToolService/modules/JobRouting.test.ts b/server/services/manualReviewToolService/modules/JobRouting.test.ts index ff804096..eca6eb3d 100644 --- a/server/services/manualReviewToolService/modules/JobRouting.test.ts +++ b/server/services/manualReviewToolService/modules/JobRouting.test.ts @@ -1,4 +1,3 @@ -/* eslint-disable max-lines */ import { ScalarTypes } from '@roostorg/coop-types'; import { uid } from 'uid'; diff --git a/server/services/manualReviewToolService/modules/ReporterInvalidation.test.ts b/server/services/manualReviewToolService/modules/ReporterInvalidation.test.ts index df8d7e2a..e7a745b7 100644 --- a/server/services/manualReviewToolService/modules/ReporterInvalidation.test.ts +++ b/server/services/manualReviewToolService/modules/ReporterInvalidation.test.ts @@ -1,4 +1,3 @@ -/* eslint-disable max-lines */ import { uid } from 'uid'; import { v1 as uuidv1 } from 'uuid'; diff --git a/server/services/moderationConfigService/moderationConfigService.test.ts b/server/services/moderationConfigService/moderationConfigService.test.ts index bcaa9527..4447c8e0 100644 --- a/server/services/moderationConfigService/moderationConfigService.test.ts +++ b/server/services/moderationConfigService/moderationConfigService.test.ts @@ -1,4 +1,3 @@ -/* eslint-disable max-lines */ import { faker } from '@faker-js/faker'; import { Kysely } from 'kysely'; import { type UnionToIntersection } from 'type-fest'; diff --git a/server/services/ncmecService/dbTypes.ts b/server/services/ncmecService/dbTypes.ts index 8522b6d4..a26c57f7 100644 --- a/server/services/ncmecService/dbTypes.ts +++ b/server/services/ncmecService/dbTypes.ts @@ -44,7 +44,7 @@ export type NcmecReportingServicePg = { report_id: string; user_id: string; user_item_type_id: string; - reported_media: NonEmptyArray; + reported_media: Array; reviewer_id?: string; created_at: GeneratedAlways; updated_at: GeneratedAlways; diff --git a/server/services/ncmecService/ncmecEnqueueToMrt.ts b/server/services/ncmecService/ncmecEnqueueToMrt.ts index 7fa771fe..f3488a3c 100644 --- a/server/services/ncmecService/ncmecEnqueueToMrt.ts +++ b/server/services/ncmecService/ncmecEnqueueToMrt.ts @@ -177,10 +177,6 @@ export default class NcmecEnqueueToMrt { reportedItemType, ); - if (allMediaItems.length === 0) { - return { status: 'SKIPPED' }; - } - // TODO: Write this to a data warehouse table and enqueue based off of a job instead await this.manualReviewToolService.enqueue( { diff --git a/server/services/ncmecService/ncmecReporting.test.ts b/server/services/ncmecService/ncmecReporting.test.ts index 5daa0fd3..5f19a320 100644 --- a/server/services/ncmecService/ncmecReporting.test.ts +++ b/server/services/ncmecService/ncmecReporting.test.ts @@ -1,6 +1,7 @@ import { buildInternetDetailsFromOrgSetting, clampIncidentDateTimeToPast, + latestEvidenceTimestamp, mergeFieldRoleIpIntoEvents, NCMECEvent, resolveReportedPersonEmail, @@ -143,7 +144,80 @@ describe('NCMEC reporting', () => { it('throws on invalid timestamps', () => { expect(() => clampIncidentDateTimeToPast('not-a-date', NOW_MS)).toThrow( - /Invalid media createdAt timestamp/, + /Invalid timestamp for incidentDateTime/, + ); + }); + }); + + describe('latestEvidenceTimestamp', () => { + const media = (createdAt: string) => ({ createdAt }); + const thread = (...sentAts: (string | Date)[]) => ({ + reportedContent: sentAts.map((sentAt) => ({ sentAt })), + }); + + it('returns the most recent media createdAt', () => { + expect( + latestEvidenceTimestamp( + [ + media('2026-01-10T00:00:00.000Z'), + media('2026-01-12T00:00:00.000Z'), + ], + [], + ), + ).toEqual('2026-01-12T00:00:00.000Z'); + }); + + it('derives the timestamp from messages for a text-only report', () => { + expect( + latestEvidenceTimestamp( + [], + [thread('2026-01-05T00:00:00.000Z', '2026-01-08T00:00:00.000Z')], + ), + ).toEqual('2026-01-08T00:00:00.000Z'); + }); + + it('takes the max across both media and messages', () => { + expect( + latestEvidenceTimestamp( + [media('2026-01-12T00:00:00.000Z')], + [thread('2026-01-20T00:00:00.000Z')], + ), + ).toEqual('2026-01-20T00:00:00.000Z'); + }); + + it('accepts Date-valued message timestamps', () => { + expect( + latestEvidenceTimestamp( + [], + [thread(new Date('2026-01-09T00:00:00.000Z'))], + ), + ).toEqual('2026-01-09T00:00:00.000Z'); + }); + + it('throws when there is no evidence at all', () => { + expect(() => latestEvidenceTimestamp([], [])).toThrow( + /Report has neither media nor messages/, + ); + expect(() => latestEvidenceTimestamp([], [thread()])).toThrow( + /Report has neither media nor messages/, + ); + }); + + it('skips an unparseable timestamp and uses the latest valid one', () => { + expect( + latestEvidenceTimestamp( + [], + [thread('not-a-date', '2026-01-08T00:00:00.000Z')], + ), + ).toEqual('2026-01-08T00:00:00.000Z'); + }); + + it('throws when evidence exists but no timestamp parses', () => { + expect(() => latestEvidenceTimestamp([media('not-a-date')], [])).toThrow( + /Invalid timestamp for incidentDateTime/, + ); + expect(() => latestEvidenceTimestamp([], [thread('not-a-date')])).toThrow( + /Invalid timestamp for incidentDateTime/, ); }); }); diff --git a/server/services/ncmecService/ncmecReporting.ts b/server/services/ncmecService/ncmecReporting.ts index a660b816..e2166755 100644 --- a/server/services/ncmecService/ncmecReporting.ts +++ b/server/services/ncmecService/ncmecReporting.ts @@ -13,10 +13,7 @@ import { type JSONSchemaV4 } from '../../utils/json-schema-types.js'; import { type FixKyselyRowCorrelation } from '../../utils/kysely.js'; import { logErrorJson } from '../../utils/logging.js'; import { assertUnreachable, withRetries } from '../../utils/misc.js'; -import { - type CollapseCases, - type NonEmptyArray, -} from '../../utils/typescript-types.js'; +import { type CollapseCases } from '../../utils/typescript-types.js'; import { rawItemSubmissionToItemSubmission } from '../itemProcessingService/makeItemSubmission.js'; import { type RawItemData } from '../itemProcessingService/toNormalizedItemDataOrErrors.js'; import { @@ -512,9 +509,7 @@ export function clampIncidentDateTimeToPast( ): { value: string; wasClamped: boolean } { const maxCreatedAtMs = new Date(maxCreatedAt).getTime(); if (Number.isNaN(maxCreatedAtMs)) { - throw new Error( - `Invalid media createdAt timestamp for incidentDateTime: ${maxCreatedAt}`, - ); + throw new Error(`Invalid timestamp for incidentDateTime: ${maxCreatedAt}`); } const ceilingMs = nowMs - 1000; const wasClamped = maxCreatedAtMs > ceilingMs; @@ -525,6 +520,32 @@ export function clampIncidentDateTimeToPast( }; } +// consolidate timestamp fetching and verification in one function for media and +// threads +export function latestEvidenceTimestamp( + media: readonly { createdAt: string }[], + threads: readonly { + reportedContent: readonly { sentAt: string | Date }[]; + }[], +): string { + const rawTimestamps: (string | Date)[] = [ + ...media.map((m) => m.createdAt), + ...threads.flatMap((t) => t.reportedContent.map((c) => c.sentAt)), + ]; + if (rawTimestamps.length === 0) { + throw new Error('Report has neither media nor messages'); + } + // updated to allow a lenient approach to timestamps. If none parse, throw an + // error and surface the bad data via the validation error path + const evidenceTimestampsMs = rawTimestamps + .map((raw) => (raw instanceof Date ? raw.getTime() : Date.parse(raw))) + .filter((ms) => !Number.isNaN(ms)); + if (evidenceTimestampsMs.length === 0) { + throw new Error('Invalid timestamp for incidentDateTime'); + } + return new Date(Math.max(...evidenceTimestampsMs)).toISOString(); +} + /** Rebuild an ipCaptureEvent object with keys in NCMEC XSD `xs:sequence` * order (ipAddress, eventName, dateTime, possibleProxy, port) so xml-js * serialises the children in the order NCMEC's validator requires. Webhook @@ -1555,6 +1576,7 @@ export default class NcmecReporting { } if ( + reportedMedia.length > 0 && responseBody.media?.filter( (it) => it.missing === false || it.missing === undefined, ).length === 0 @@ -1801,22 +1823,10 @@ export default class NcmecReporting { ); } - if (reportParams.media.length === 0) { - throw new Error('No media in report'); - } - const latestMedia = _.maxBy(reportParams.media, (m) => { - const ms = Date.parse(m.createdAt); - if (Number.isNaN(ms)) { - throw new Error( - `Invalid media createdAt timestamp for incidentDateTime: ${m.createdAt}`, - ); - } - return ms; - }); - if (latestMedia === undefined) { - throw new Error('No media in report'); - } - const maxCreatedAt = latestMedia.createdAt; + const maxCreatedAt = latestEvidenceTimestamp( + reportParams.media, + reportParams.threads, + ); const { value: clampedIncidentDateTime, wasClamped } = clampIncidentDateTimeToPast(maxCreatedAt); @@ -2039,8 +2049,7 @@ export default class NcmecReporting { user_item_type_id: reportParams.reportedUser.typeId, reviewer_id: reportParams.reviewerId, - // Safe to cast as a non empty array because of the createdAt check above - reported_media: reportedMedia as NonEmptyArray, + reported_media: reportedMedia, report_xml: xml, additional_files: additionalFiles, reported_messages: threadCsvs, diff --git a/server/services/ncmecService/ncmecReviewerErrors.test.ts b/server/services/ncmecService/ncmecReviewerErrors.test.ts index f521fe52..6fda1b4e 100644 --- a/server/services/ncmecService/ncmecReviewerErrors.test.ts +++ b/server/services/ncmecService/ncmecReviewerErrors.test.ts @@ -35,8 +35,18 @@ describe('summarizeNcmecErrorForReviewer', () => { it('passes through known reviewer-friendly local errors verbatim', () => { expect( - summarizeNcmecErrorForReviewer(new Error('No media in report')), - ).toBe('No media in report'); + summarizeNcmecErrorForReviewer( + new Error('Report has neither media nor messages'), + ), + ).toBe('Report has neither media nor messages'); + }); + + it('classifies an unparseable incidentDateTime timestamp as validation', () => { + expect( + summarizeNcmecErrorForReviewer( + new Error('Invalid timestamp for incidentDateTime: not-a-date'), + ), + ).toMatch(/failed validation/); }); it('classifies missing-config throws to a config category', () => { diff --git a/server/services/ncmecService/ncmecReviewerErrors.ts b/server/services/ncmecService/ncmecReviewerErrors.ts index 8bf8dcf9..299f5a11 100644 --- a/server/services/ncmecService/ncmecReviewerErrors.ts +++ b/server/services/ncmecService/ncmecReviewerErrors.ts @@ -17,7 +17,7 @@ const REVIEWER_ERROR_MESSAGES = { // Allowlist of thrown messages that are already operator-friendly. New // throw sites stay opaque until classified explicitly. const ALREADY_REVIEWER_FRIENDLY: ReadonlySet = new Set([ - 'No media in report', + 'Report has neither media nor messages', 'Organization does not have a NCMEC preservation endpoint', 'NCMEC report requires a non-empty reporter contact email; configure it in Settings → NCMEC.', 'escalateToHighPriority must be non-blank when supplied and at most 3000 characters', @@ -34,7 +34,7 @@ const REVIEWER_PREFIX_RULES: readonly { { prefix: 'org id not found', category: 'CONFIG' }, { prefix: 'Unable to find reported media in job payload', category: 'MEDIA' }, { prefix: 'Unable to find item type for reported media', category: 'MEDIA' }, - { prefix: 'Invalid media createdAt timestamp', category: 'MEDIA' }, + { prefix: 'Invalid timestamp for incidentDateTime', category: 'VALIDATION' }, { prefix: 'Cannot download media from', category: 'MEDIA' }, { prefix: 'NCMEC file upload failed', category: 'MEDIA' }, { prefix: 'NCMEC thread CSV upload failed', category: 'MEDIA' },