Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
/* eslint-disable max-lines */
import { uid } from 'uid';
import { v1 as uuidv1 } from 'uuid';

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
/* eslint-disable max-lines */
import { ScalarTypes } from '@roostorg/coop-types';
import { uid } from 'uid';

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
/* eslint-disable max-lines */
import { uid } from 'uid';
import { v1 as uuidv1 } from 'uuid';

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
/* eslint-disable max-lines */
import { faker } from '@faker-js/faker';
import { Kysely } from 'kysely';
import { type UnionToIntersection } from 'type-fest';
Expand Down
2 changes: 1 addition & 1 deletion server/services/ncmecService/dbTypes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ export type NcmecReportingServicePg = {
report_id: string;
user_id: string;
user_item_type_id: string;
reported_media: NonEmptyArray<NcmecMediaReport>;
reported_media: Array<NcmecMediaReport>;
reviewer_id?: string;
created_at: GeneratedAlways<Date>;
updated_at: GeneratedAlways<Date>;
Expand Down
4 changes: 0 additions & 4 deletions server/services/ncmecService/ncmecEnqueueToMrt.ts
Original file line number Diff line number Diff line change
Expand Up @@ -177,10 +177,6 @@ export default class NcmecEnqueueToMrt {
reportedItemType,
);

if (allMediaItems.length === 0) {
return { status: 'SKIPPED' };
}

// TODO: Write this to a data warehouse table and enqueue based off of a job instead
await this.manualReviewToolService.enqueue(
{
Expand Down
76 changes: 75 additions & 1 deletion server/services/ncmecService/ncmecReporting.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import {
buildInternetDetailsFromOrgSetting,
clampIncidentDateTimeToPast,
latestEvidenceTimestamp,
mergeFieldRoleIpIntoEvents,
NCMECEvent,
resolveReportedPersonEmail,
Expand Down Expand Up @@ -143,7 +144,80 @@ describe('NCMEC reporting', () => {

it('throws on invalid timestamps', () => {
expect(() => clampIncidentDateTimeToPast('not-a-date', NOW_MS)).toThrow(
/Invalid media createdAt timestamp/,
/Invalid timestamp for incidentDateTime/,
);
});
});

describe('latestEvidenceTimestamp', () => {
const media = (createdAt: string) => ({ createdAt });
const thread = (...sentAts: (string | Date)[]) => ({
reportedContent: sentAts.map((sentAt) => ({ sentAt })),
});

it('returns the most recent media createdAt', () => {
expect(
latestEvidenceTimestamp(
[
media('2026-01-10T00:00:00.000Z'),
media('2026-01-12T00:00:00.000Z'),
],
[],
),
).toEqual('2026-01-12T00:00:00.000Z');
});

it('derives the timestamp from messages for a text-only report', () => {
expect(
latestEvidenceTimestamp(
[],
[thread('2026-01-05T00:00:00.000Z', '2026-01-08T00:00:00.000Z')],
),
).toEqual('2026-01-08T00:00:00.000Z');
});

it('takes the max across both media and messages', () => {
expect(
latestEvidenceTimestamp(
[media('2026-01-12T00:00:00.000Z')],
[thread('2026-01-20T00:00:00.000Z')],
),
).toEqual('2026-01-20T00:00:00.000Z');
});

it('accepts Date-valued message timestamps', () => {
expect(
latestEvidenceTimestamp(
[],
[thread(new Date('2026-01-09T00:00:00.000Z'))],
),
).toEqual('2026-01-09T00:00:00.000Z');
});

it('throws when there is no evidence at all', () => {
expect(() => latestEvidenceTimestamp([], [])).toThrow(
/Report has neither media nor messages/,
);
expect(() => latestEvidenceTimestamp([], [thread()])).toThrow(
/Report has neither media nor messages/,
);
});

it('skips an unparseable timestamp and uses the latest valid one', () => {
expect(
latestEvidenceTimestamp(
[],
[thread('not-a-date', '2026-01-08T00:00:00.000Z')],
),
).toEqual('2026-01-08T00:00:00.000Z');
});

it('throws when evidence exists but no timestamp parses', () => {
expect(() => latestEvidenceTimestamp([media('not-a-date')], [])).toThrow(
/Invalid timestamp for incidentDateTime/,
);
expect(() => latestEvidenceTimestamp([], [thread('not-a-date')])).toThrow(
/Invalid timestamp for incidentDateTime/,
);
});
});
Expand Down
59 changes: 34 additions & 25 deletions server/services/ncmecService/ncmecReporting.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,7 @@ import { type JSONSchemaV4 } from '../../utils/json-schema-types.js';
import { type FixKyselyRowCorrelation } from '../../utils/kysely.js';
import { logErrorJson } from '../../utils/logging.js';
import { assertUnreachable, withRetries } from '../../utils/misc.js';
import {
type CollapseCases,
type NonEmptyArray,
} from '../../utils/typescript-types.js';
import { type CollapseCases } from '../../utils/typescript-types.js';
import { rawItemSubmissionToItemSubmission } from '../itemProcessingService/makeItemSubmission.js';
import { type RawItemData } from '../itemProcessingService/toNormalizedItemDataOrErrors.js';
import {
Expand Down Expand Up @@ -512,9 +509,7 @@ export function clampIncidentDateTimeToPast(
): { value: string; wasClamped: boolean } {
const maxCreatedAtMs = new Date(maxCreatedAt).getTime();
if (Number.isNaN(maxCreatedAtMs)) {
throw new Error(
Comment thread
calebmcquaid marked this conversation as resolved.
`Invalid media createdAt timestamp for incidentDateTime: ${maxCreatedAt}`,
);
throw new Error(`Invalid timestamp for incidentDateTime: ${maxCreatedAt}`);
}
const ceilingMs = nowMs - 1000;
const wasClamped = maxCreatedAtMs > ceilingMs;
Expand All @@ -525,6 +520,32 @@ export function clampIncidentDateTimeToPast(
};
}

// consolidate timestamp fetching and verification in one function for media and
// threads
export function latestEvidenceTimestamp(
media: readonly { createdAt: string }[],
threads: readonly {
reportedContent: readonly { sentAt: string | Date }[];
}[],
): string {
const rawTimestamps: (string | Date)[] = [
...media.map((m) => m.createdAt),
...threads.flatMap((t) => t.reportedContent.map((c) => c.sentAt)),
];
if (rawTimestamps.length === 0) {
throw new Error('Report has neither media nor messages');
}
// updated to allow a lenient approach to timestamps. If none parse, throw an

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
// updated to allow a lenient approach to timestamps. If none parse, throw an
// allow a lenient approach to timestamps. If none parse, throw an

code comments should explain the current state of the code, not what was! as a reader i see this and think "updated from what?" -- but more importantly that question is just not important for the reader's understanding of the codebase, so it's only a distraction

// error and surface the bad data via the validation error path
const evidenceTimestampsMs = rawTimestamps

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what's an example where an invalid date string might end up here? how might that happen?

(i am wondering if this handling is necessary or if it's unnecessarily defensive).

.map((raw) => (raw instanceof Date ? raw.getTime() : Date.parse(raw)))
.filter((ms) => !Number.isNaN(ms));
if (evidenceTimestampsMs.length === 0) {
throw new Error('Invalid timestamp for incidentDateTime');
}
Comment on lines +538 to +545
return new Date(Math.max(...evidenceTimestampsMs)).toISOString();
}

/** Rebuild an ipCaptureEvent object with keys in NCMEC XSD `xs:sequence`
* order (ipAddress, eventName, dateTime, possibleProxy, port) so xml-js
* serialises the children in the order NCMEC's validator requires. Webhook
Expand Down Expand Up @@ -1555,6 +1576,7 @@ export default class NcmecReporting {
}

if (
reportedMedia.length > 0 &&
responseBody.media?.filter(
(it) => it.missing === false || it.missing === undefined,
).length === 0
Expand Down Expand Up @@ -1801,22 +1823,10 @@ export default class NcmecReporting {
);
}

if (reportParams.media.length === 0) {
throw new Error('No media in report');
}
const latestMedia = _.maxBy(reportParams.media, (m) => {
const ms = Date.parse(m.createdAt);
if (Number.isNaN(ms)) {
throw new Error(
`Invalid media createdAt timestamp for incidentDateTime: ${m.createdAt}`,
);
}
return ms;
});
if (latestMedia === undefined) {
throw new Error('No media in report');
}
const maxCreatedAt = latestMedia.createdAt;
const maxCreatedAt = latestEvidenceTimestamp(
reportParams.media,
reportParams.threads,
);
Comment thread
coderabbitai[bot] marked this conversation as resolved.

const { value: clampedIncidentDateTime, wasClamped } =
clampIncidentDateTimeToPast(maxCreatedAt);
Expand Down Expand Up @@ -2039,8 +2049,7 @@ export default class NcmecReporting {

user_item_type_id: reportParams.reportedUser.typeId,
reviewer_id: reportParams.reviewerId,
// Safe to cast as a non empty array because of the createdAt check above
reported_media: reportedMedia as NonEmptyArray<NcmecMediaReport>,
reported_media: reportedMedia,
report_xml: xml,
additional_files: additionalFiles,
reported_messages: threadCsvs,
Expand Down
14 changes: 12 additions & 2 deletions server/services/ncmecService/ncmecReviewerErrors.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,18 @@ describe('summarizeNcmecErrorForReviewer', () => {

it('passes through known reviewer-friendly local errors verbatim', () => {
expect(
summarizeNcmecErrorForReviewer(new Error('No media in report')),
).toBe('No media in report');
summarizeNcmecErrorForReviewer(
new Error('Report has neither media nor messages'),
),
).toBe('Report has neither media nor messages');
});

it('classifies an unparseable incidentDateTime timestamp as validation', () => {
expect(
summarizeNcmecErrorForReviewer(
new Error('Invalid timestamp for incidentDateTime: not-a-date'),
),
).toMatch(/failed validation/);
});

it('classifies missing-config throws to a config category', () => {
Expand Down
4 changes: 2 additions & 2 deletions server/services/ncmecService/ncmecReviewerErrors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ const REVIEWER_ERROR_MESSAGES = {
// Allowlist of thrown messages that are already operator-friendly. New
// throw sites stay opaque until classified explicitly.
const ALREADY_REVIEWER_FRIENDLY: ReadonlySet<string> = new Set([
'No media in report',
'Report has neither media nor messages',
'Organization does not have a NCMEC preservation endpoint',
'NCMEC report requires a non-empty reporter contact email; configure it in Settings → NCMEC.',
'escalateToHighPriority must be non-blank when supplied and at most 3000 characters',
Expand All @@ -34,7 +34,7 @@ const REVIEWER_PREFIX_RULES: readonly {
{ prefix: 'org id not found', category: 'CONFIG' },
{ prefix: 'Unable to find reported media in job payload', category: 'MEDIA' },
{ prefix: 'Unable to find item type for reported media', category: 'MEDIA' },
{ prefix: 'Invalid media createdAt timestamp', category: 'MEDIA' },
{ prefix: 'Invalid timestamp for incidentDateTime', category: 'VALIDATION' },
{ prefix: 'Cannot download media from', category: 'MEDIA' },
{ prefix: 'NCMEC file upload failed', category: 'MEDIA' },
{ prefix: 'NCMEC thread CSV upload failed', category: 'MEDIA' },
Expand Down
Loading