Skip to content

Commit 589b03f

Browse files
committed
fix(core): truncate large error stacks and messages to prevent OOM
1 parent ff290df commit 589b03f

5 files changed

Lines changed: 246 additions & 21 deletions

File tree

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@trigger.dev/core": patch
3+
---
4+
5+
Truncate large error stacks and messages to prevent OOM crashes. Stack traces are capped at 50 frames (keeping top 5 + bottom 45 with an omission notice), individual stack lines at 1024 chars, and error messages at 1000 chars. Applied in parseError, sanitizeError, and OTel span recording.

packages/core/src/v3/errors.ts

Lines changed: 62 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -154,22 +154,73 @@ export function isCompleteTaskWithOutput(error: unknown): error is CompleteTaskW
154154
return error instanceof Error && error.name === "CompleteTaskWithOutput";
155155
}
156156

157+
const MAX_STACK_FRAMES = 50;
158+
const KEEP_TOP_FRAMES = 5;
159+
const MAX_STACK_LINE_LENGTH = 1024;
160+
const MAX_MESSAGE_LENGTH = 1_000;
161+
162+
/** Truncate a stack trace to at most MAX_STACK_FRAMES frames, keeping
163+
* the top (closest to throw) and bottom (entry points) frames. */
164+
export function truncateStack(stack: string | undefined): string {
165+
if (!stack) return "";
166+
167+
const lines = stack.split("\n");
168+
169+
// First line(s) before the first frame are the error message
170+
const messageLines: string[] = [];
171+
const frameLines: string[] = [];
172+
173+
for (const line of lines) {
174+
if (frameLines.length === 0 && !line.trimStart().startsWith("at ")) {
175+
messageLines.push(line);
176+
} else {
177+
// Truncate individual lines to prevent regex DoS in downstream parsers
178+
frameLines.push(
179+
line.length > MAX_STACK_LINE_LENGTH
180+
? line.slice(0, MAX_STACK_LINE_LENGTH) + "...[truncated]"
181+
: line
182+
);
183+
}
184+
}
185+
186+
if (frameLines.length <= MAX_STACK_FRAMES) {
187+
return [...messageLines, ...frameLines].join("\n");
188+
}
189+
190+
const keepBottom = MAX_STACK_FRAMES - KEEP_TOP_FRAMES;
191+
const omitted = frameLines.length - MAX_STACK_FRAMES;
192+
193+
return [
194+
...messageLines,
195+
...frameLines.slice(0, KEEP_TOP_FRAMES),
196+
` ... ${omitted} frames omitted ...`,
197+
...frameLines.slice(-keepBottom),
198+
].join("\n");
199+
}
200+
201+
export function truncateMessage(message: string | undefined): string {
202+
if (!message) return "";
203+
return message.length > MAX_MESSAGE_LENGTH
204+
? message.slice(0, MAX_MESSAGE_LENGTH) + "...[truncated]"
205+
: message;
206+
}
207+
157208
export function parseError(error: unknown): TaskRunError {
158209
if (isInternalError(error)) {
159210
return {
160211
type: "INTERNAL_ERROR",
161212
code: error.code,
162-
message: error.message,
163-
stackTrace: error.stack ?? "",
213+
message: truncateMessage(error.message),
214+
stackTrace: truncateStack(error.stack),
164215
};
165216
}
166217

167218
if (error instanceof Error) {
168219
return {
169220
type: "BUILT_IN_ERROR",
170221
name: error.name,
171-
message: error.message,
172-
stackTrace: error.stack ?? "",
222+
message: truncateMessage(error.message),
223+
stackTrace: truncateStack(error.stack),
173224
};
174225
}
175226

@@ -248,35 +299,35 @@ export function createJsonErrorObject(error: TaskRunError): SerializedError {
248299
}
249300
}
250301

251-
// Removes any null characters from the error message
302+
// Removes null characters and truncates oversized fields to prevent OOM
252303
export function sanitizeError(error: TaskRunError): TaskRunError {
253304
switch (error.type) {
254305
case "BUILT_IN_ERROR": {
255306
return {
256307
type: "BUILT_IN_ERROR",
257-
message: error.message?.replace(/\0/g, ""),
308+
message: truncateMessage(error.message?.replace(/\0/g, "")),
258309
name: error.name?.replace(/\0/g, ""),
259-
stackTrace: error.stackTrace?.replace(/\0/g, ""),
310+
stackTrace: truncateStack(error.stackTrace?.replace(/\0/g, "")),
260311
};
261312
}
262313
case "STRING_ERROR": {
263314
return {
264315
type: "STRING_ERROR",
265-
raw: error.raw.replace(/\0/g, ""),
316+
raw: truncateMessage(error.raw.replace(/\0/g, "")),
266317
};
267318
}
268319
case "CUSTOM_ERROR": {
269320
return {
270321
type: "CUSTOM_ERROR",
271-
raw: error.raw.replace(/\0/g, ""),
322+
raw: truncateMessage(error.raw.replace(/\0/g, "")),
272323
};
273324
}
274325
case "INTERNAL_ERROR": {
275326
return {
276327
type: "INTERNAL_ERROR",
277328
code: error.code,
278-
message: error.message?.replace(/\0/g, ""),
279-
stackTrace: error.stackTrace?.replace(/\0/g, ""),
329+
message: truncateMessage(error.message?.replace(/\0/g, "")),
330+
stackTrace: truncateStack(error.stackTrace?.replace(/\0/g, "")),
280331
};
281332
}
282333
}

packages/core/src/v3/otel/utils.ts

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,30 @@
11
import { type Span, SpanStatusCode, context, propagation } from "@opentelemetry/api";
2+
import { truncateStack, truncateMessage } from "../errors.js";
3+
4+
const MAX_GENERIC_LENGTH = 5_000;
25

36
export function recordSpanException(span: Span, error: unknown) {
47
if (error instanceof Error) {
58
span.recordException(sanitizeSpanError(error));
69
} else if (typeof error === "string") {
7-
span.recordException(error.replace(/\0/g, ""));
10+
const clean = error.replace(/\0/g, "");
11+
span.recordException(
12+
clean.length > MAX_GENERIC_LENGTH ? clean.slice(0, MAX_GENERIC_LENGTH) + "...[truncated]" : clean
13+
);
814
} else {
9-
span.recordException(JSON.stringify(error).replace(/\0/g, ""));
15+
const json = JSON.stringify(error).replace(/\0/g, "");
16+
span.recordException(
17+
json.length > MAX_GENERIC_LENGTH ? json.slice(0, MAX_GENERIC_LENGTH) + "...[truncated]" : json
18+
);
1019
}
1120

1221
span.setStatus({ code: SpanStatusCode.ERROR });
1322
}
1423

1524
function sanitizeSpanError(error: Error) {
16-
// Create a new error object with the same name, message and stack trace
17-
const sanitizedError = new Error(error.message.replace(/\0/g, ""));
25+
const sanitizedError = new Error(truncateMessage(error.message.replace(/\0/g, "")));
1826
sanitizedError.name = error.name.replace(/\0/g, "");
19-
sanitizedError.stack = error.stack?.replace(/\0/g, "");
27+
sanitizedError.stack = truncateStack(error.stack?.replace(/\0/g, "")) || undefined;
2028

2129
return sanitizedError;
2230
}

packages/core/src/v3/tracer.ts

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -145,11 +145,7 @@ export class TriggerTracer {
145145
}
146146

147147
if (!spanEnded) {
148-
if (typeof e === "string" || e instanceof Error) {
149-
span.recordException(e);
150-
}
151-
152-
span.setStatus({ code: SpanStatusCode.ERROR });
148+
recordSpanException(span, e);
153149
}
154150

155151
throw e;

packages/core/test/errors.test.ts

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
import { describe, it, expect } from "vitest";
2+
import { truncateStack, truncateMessage, parseError, sanitizeError } from "../src/v3/errors.js";
3+
4+
// Helper: build a fake stack with N frames
5+
function buildStack(messageLines: string[], frameCount: number): string {
6+
const frames = Array.from(
7+
{ length: frameCount },
8+
(_, i) => ` at functionName${i} (/path/to/file${i}.ts:${i + 1}:${i + 10})`
9+
);
10+
return [...messageLines, ...frames].join("\n");
11+
}
12+
13+
describe("truncateStack", () => {
14+
it("returns empty string for undefined", () => {
15+
expect(truncateStack(undefined)).toBe("");
16+
});
17+
18+
it("returns empty string for empty string", () => {
19+
expect(truncateStack("")).toBe("");
20+
});
21+
22+
it("preserves a short stack unchanged", () => {
23+
const stack = buildStack(["Error: something broke"], 10);
24+
expect(truncateStack(stack)).toBe(stack);
25+
});
26+
27+
it("preserves exactly 50 frames", () => {
28+
const stack = buildStack(["Error: at the limit"], 50);
29+
const result = truncateStack(stack);
30+
expect(result).toBe(stack);
31+
expect(result.split("\n").filter((l) => l.trimStart().startsWith("at ")).length).toBe(50);
32+
});
33+
34+
it("truncates to 50 frames when exceeding the limit", () => {
35+
const stack = buildStack(["Error: too many frames"], 200);
36+
const result = truncateStack(stack);
37+
const lines = result.split("\n");
38+
39+
// Message line + 5 top + 1 omitted notice + 45 bottom = 52 lines
40+
expect(lines[0]).toBe("Error: too many frames");
41+
expect(lines).toContain(" ... 150 frames omitted ...");
42+
43+
const frameLines = lines.filter((l) => l.trimStart().startsWith("at "));
44+
expect(frameLines.length).toBe(50);
45+
46+
// First kept frame is frame 0 (top of stack)
47+
expect(frameLines[0]).toContain("functionName0");
48+
// Last kept frame is the last original frame
49+
expect(frameLines[frameLines.length - 1]).toContain("functionName199");
50+
});
51+
52+
it("preserves multi-line error messages before frames", () => {
53+
const stack = buildStack(["TypeError: cannot read property", " caused by: something"], 60);
54+
const result = truncateStack(stack);
55+
const lines = result.split("\n");
56+
57+
expect(lines[0]).toBe("TypeError: cannot read property");
58+
expect(lines[1]).toBe(" caused by: something");
59+
expect(lines).toContain(" ... 10 frames omitted ...");
60+
});
61+
62+
it("truncates individual lines longer than 1024 chars", () => {
63+
const longFrame = ` at someFn (${"x".repeat(2000)}:1:1)`;
64+
const stack = ["Error: long line", longFrame].join("\n");
65+
const result = truncateStack(stack);
66+
const frameLine = result.split("\n")[1]!;
67+
68+
expect(frameLine.length).toBeLessThan(1100);
69+
expect(frameLine).toContain("...[truncated]");
70+
});
71+
});
72+
73+
describe("truncateMessage", () => {
74+
it("returns empty string for undefined", () => {
75+
expect(truncateMessage(undefined)).toBe("");
76+
});
77+
78+
it("returns empty string for empty string", () => {
79+
expect(truncateMessage("")).toBe("");
80+
});
81+
82+
it("preserves a short message", () => {
83+
expect(truncateMessage("hello")).toBe("hello");
84+
});
85+
86+
it("truncates messages over 1000 chars", () => {
87+
const long = "x".repeat(5000);
88+
const result = truncateMessage(long);
89+
expect(result.length).toBeLessThan(1100);
90+
expect(result).toContain("...[truncated]");
91+
});
92+
93+
it("preserves a message at exactly 1000 chars", () => {
94+
const exact = "x".repeat(1000);
95+
expect(truncateMessage(exact)).toBe(exact);
96+
});
97+
});
98+
99+
describe("parseError truncation", () => {
100+
it("truncates large stack traces in Error objects", () => {
101+
const error = new Error("boom");
102+
error.stack = buildStack(["Error: boom"], 200);
103+
const parsed = parseError(error);
104+
105+
expect(parsed.type).toBe("BUILT_IN_ERROR");
106+
if (parsed.type === "BUILT_IN_ERROR") {
107+
const frameLines = parsed.stackTrace.split("\n").filter((l) => l.trimStart().startsWith("at "));
108+
expect(frameLines.length).toBe(50);
109+
expect(parsed.stackTrace).toContain("frames omitted");
110+
}
111+
});
112+
113+
it("truncates large error messages", () => {
114+
const error = new Error("x".repeat(5000));
115+
const parsed = parseError(error);
116+
117+
if (parsed.type === "BUILT_IN_ERROR") {
118+
expect(parsed.message.length).toBeLessThan(1100);
119+
expect(parsed.message).toContain("...[truncated]");
120+
}
121+
});
122+
});
123+
124+
describe("sanitizeError truncation", () => {
125+
it("truncates stack traces during sanitization", () => {
126+
const result = sanitizeError({
127+
type: "BUILT_IN_ERROR",
128+
name: "Error",
129+
message: "boom",
130+
stackTrace: buildStack(["Error: boom"], 200),
131+
});
132+
133+
if (result.type === "BUILT_IN_ERROR") {
134+
const frameLines = result.stackTrace.split("\n").filter((l) => l.trimStart().startsWith("at "));
135+
expect(frameLines.length).toBe(50);
136+
}
137+
});
138+
139+
it("strips null bytes and truncates", () => {
140+
const result = sanitizeError({
141+
type: "BUILT_IN_ERROR",
142+
name: "Error\0",
143+
message: "hello\0world",
144+
stackTrace: "Error: hello\0world\n at fn (/path.ts:1:1)",
145+
});
146+
147+
if (result.type === "BUILT_IN_ERROR") {
148+
expect(result.name).toBe("Error");
149+
expect(result.message).toBe("helloworld");
150+
expect(result.stackTrace).not.toContain("\0");
151+
}
152+
});
153+
154+
it("truncates STRING_ERROR raw field", () => {
155+
const result = sanitizeError({
156+
type: "STRING_ERROR",
157+
raw: "x".repeat(5000),
158+
});
159+
160+
if (result.type === "STRING_ERROR") {
161+
expect(result.raw.length).toBeLessThan(1100);
162+
expect(result.raw).toContain("...[truncated]");
163+
}
164+
});
165+
});

0 commit comments

Comments
 (0)