Skip to content

feat: add Sail benchmarks and E2B audit #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
singh-pk-runable wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: add Sail benchmarks and E2B audit #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .env.template
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
E2B_API_KEY=
DAYTONA_API_KEY=
BL_WORKSPACE=runable
BL_API_KEY
BL_API_KEY=
SAIL_API_KEY=
26 changes: 24 additions & 2 deletions bun.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

259 changes: 259 additions & 0 deletions e2b-audit.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,259 @@
import { Sandbox as E2BSandbox } from "@e2b/desktop";
import { createHash } from "node:crypto";

function stats(values: number[]) {
const mean = values.reduce((a, b) => a + b, 0) / values.length;
return { mean, min: Math.min(...values), max: Math.max(...values) };
}

function report(label: string, times: number[]) {
const s = stats(times);
console.log(
` ${label.padEnd(46)} avg: ${s.mean.toFixed(1)}ms min: ${s.min.toFixed(1)}ms max: ${s.max.toFixed(1)}ms (${times.length} runs)`
);
}

const sha256 = (data: string | Uint8Array) =>
createHash("sha256").update(data).digest("hex");

const randomBytes = (n: number) => {
const buf = new Uint8Array(n);
crypto.getRandomValues(buf);
return buf;
};

async function expectWrite(sandbox: E2BSandbox, path: string, expected: string) {
const got = await sandbox.files.read(path, { format: "text" });
if (got !== expected) throw new Error(`integrity FAIL ${path}`);
}

async function main() {
console.log("creating warm sandbox...");
const createStart = performance.now();
const sandbox = await E2BSandbox.create({ timeoutMs: 900_000 });
console.log(`warm sandbox created in ${(performance.now() - createStart).toFixed(0)}ms\n`);

try {
// ---------- 1. Latency floor ----------
console.log("== 1. Latency floor ==");
let start = performance.now();
await sandbox.commands.run("true");
console.log(` first commands.run('true') on fresh sandbox: ${(performance.now() - start).toFixed(1)}ms`);

const trueTimes: number[] = [];
for (let i = 0; i < 10; i++) {
start = performance.now();
const r = await sandbox.commands.run("true");
if (r.exitCode !== 0) throw new Error("true failed");
trueTimes.push(performance.now() - start);
}
report("commands.run('true') subsequent", trueTimes);

// raw HTTPS RTT probes against envd
const healthUrl = `https://${sandbox.getHost(49983)}/health`;
const fetchTimes: number[] = [];
let fetchOk = true;
for (let i = 0; i < 10; i++) {
start = performance.now();
try {
const res = await fetch(healthUrl);
await res.text();
} catch {
fetchOk = false;
break;
}
fetchTimes.push(performance.now() - start);
}
if (fetchOk) report("raw fetch GET /health (network RTT)", fetchTimes);
else console.log(" raw fetch GET /health failed (auth required), skipping");

const existsTimes: number[] = [];
for (let i = 0; i < 10; i++) {
start = performance.now();
await sandbox.files.exists("/");
existsTimes.push(performance.now() - start);
}
report("files.exists('/') (HTTP GET round trip)", existsTimes);

// ---------- 2. Sequential command throughput ----------
console.log("\n== 2. Sequential command throughput ==");
const echoTimes: number[] = [];
for (let i = 0; i < 10; i++) {
start = performance.now();
const r = await sandbox.commands.run("echo hello");
if (r.stdout.trim() !== "hello") throw new Error("echo mismatch");
echoTimes.push(performance.now() - start);
}
report("commands.run('echo hello') x10 hot", echoTimes);

// ---------- 3. Small-file writes ----------
console.log("\n== 3. Small-file writes (5 files, ~22 bytes each) ==");
const payload = "hello e2b file write\n";

const seqTimes: number[] = [];
for (let i = 0; i < 8; i++) {
start = performance.now();
for (let j = 0; j < 5; j++) {
await sandbox.files.write(`/tmp/s${i}_${j}.txt`, payload);
}
seqTimes.push(performance.now() - start);
}
report("5x sequential files.write", seqTimes);

const batchTimes: number[] = [];
for (let i = 0; i < 8; i++) {
const entries = Array.from({ length: 5 }, (_, j) => ({
path: `/tmp/b${i}_${j}.txt`,
data: payload,
}));
start = performance.now();
await sandbox.files.write(entries);
batchTimes.push(performance.now() - start);
}
report("1x batch files.write (5 entries)", batchTimes);

const parTimes: number[] = [];
for (let i = 0; i < 8; i++) {
start = performance.now();
await Promise.all(
Array.from({ length: 5 }, (_, j) =>
sandbox.files.write(`/tmp/p${i}_${j}.txt`, payload)
)
);
parTimes.push(performance.now() - start);
}
report("5x parallel files.write (Promise.all)", parTimes);

for (let j = 0; j < 5; j++) {
await expectWrite(sandbox, `/tmp/s0_${j}.txt`, payload);
await expectWrite(sandbox, `/tmp/b0_${j}.txt`, payload);
await expectWrite(sandbox, `/tmp/p0_${j}.txt`, payload);
}
console.log(" integrity: 15/15 small files verified");

// ---------- 4. Bulk writes ----------
console.log("\n== 4. Bulk writes ==");
const chunk64k = randomBytes(64 * 1024);
const sum64k = sha256(chunk64k);

const bulkBatchTimes: number[] = [];
for (let i = 0; i < 5; i++) {
const entries = Array.from({ length: 20 }, (_, j) => ({
path: `/tmp/bb${i}_${j}.bin`,
data: chunk64k.buffer.slice(chunk64k.byteOffset, chunk64k.byteOffset + chunk64k.byteLength) as ArrayBuffer,
}));
start = performance.now();
await sandbox.files.write(entries);
bulkBatchTimes.push(performance.now() - start);
}
report("20x64KB batch files.write (1 call)", bulkBatchTimes);

const bulkSeqTimes: number[] = [];
for (let i = 0; i < 5; i++) {
start = performance.now();
for (let j = 0; j < 20; j++) {
await sandbox.files.write(`/tmp/bs${i}_${j}.bin`, chunk64k);
}
bulkSeqTimes.push(performance.now() - start);
}
report("20x64KB sequential files.write", bulkSeqTimes);

const bulkParTimes: number[] = [];
for (let i = 0; i < 5; i++) {
start = performance.now();
await Promise.all(
Array.from({ length: 20 }, (_, j) =>
sandbox.files.write(`/tmp/bp${i}_${j}.bin`, chunk64k)
)
);
bulkParTimes.push(performance.now() - start);
}
report("20x64KB parallel files.write", bulkParTimes);

// verify a sample via in-sandbox sha256
const hashCheck = await sandbox.commands.run("sha256sum /tmp/bb0_0.bin /tmp/bs0_0.bin /tmp/bp0_0.bin");
const okCount = hashCheck.stdout.split("\n").filter((l) => l.startsWith(sum64k)).length;
if (okCount !== 3) throw new Error(`bulk integrity FAIL:\n${hashCheck.stdout}`);
console.log(" integrity: sample sha256 match on all 3 strategies");

// base64-in-argv exec write (the Sail workaround) on E2B
const b64 = Buffer.from(chunk64k).toString("base64");
const execWriteTimes: number[] = [];
for (let i = 0; i < 8; i++) {
start = performance.now();
const r = await sandbox.commands.run(`echo ${b64} | base64 -d > /tmp/ex${i}.bin`);
if (r.exitCode !== 0) throw new Error(`exec write failed: ${r.stderr}`);
execWriteTimes.push(performance.now() - start);
}
report("1x64KB via exec base64-in-argv", execWriteTimes);
const execHash = await sandbox.commands.run("sha256sum /tmp/ex0.bin");
if (!execHash.stdout.startsWith(sum64k)) throw new Error("exec write integrity FAIL");
console.log(" integrity: exec base64 write sha256 match");

try {
const cmd5 = Array.from({ length: 5 }, (_, j) => `echo ${b64} | base64 -d > /tmp/exb${j}.bin`).join(" && ");
const r = await sandbox.commands.run(cmd5);
console.log(` 5x64KB base64-in-argv single command: exit ${r.exitCode} (${(cmd5.length / 1024).toFixed(0)}KB argv)`);
} catch (e) {
console.log(` 5x64KB base64-in-argv single command FAILED: ${(e as Error).message.split("\n")[0]}`);
}

// large single files: native endpoint
for (const sizeMB of [1, 10]) {
const data = randomBytes(sizeMB * 1024 * 1024);
const sum = sha256(data);
const times: number[] = [];
for (let i = 0; i < 3; i++) {
start = performance.now();
await sandbox.files.write(`/tmp/big${sizeMB}_${i}.bin`, data);
times.push(performance.now() - start);
}
const s = stats(times);
const mbps = (sizeMB / (s.mean / 1000)).toFixed(1);
report(`files.write ${sizeMB}MB single file`, times);
console.log(` -> ${mbps} MB/s avg`);
const hr = await sandbox.commands.run(`sha256sum /tmp/big${sizeMB}_0.bin`);
if (!hr.stdout.startsWith(sum)) throw new Error(`${sizeMB}MB integrity FAIL`);
console.log(` integrity: sha256 match`);
}

// ---------- 5. Large stdout integrity ----------
console.log("\n== 5. Large stdout via commands.run ==");
for (const n of [200_000, 500_000]) {
const expected = Array.from({ length: n }, (_, i) => i + 1).join("\n") + "\n";
start = performance.now();
const r = await sandbox.commands.run(`seq 1 ${n}`, { timeoutMs: 120_000 });
const ms = performance.now() - start;
const match = r.stdout === expected;
console.log(
` seq 1 ${n}: ${ms.toFixed(0)}ms, ${r.stdout.length} bytes (expected ${expected.length}), exact match: ${match}`
);
if (!match) {
// locate first divergence
let k = 0;
while (k < Math.min(r.stdout.length, expected.length) && r.stdout[k] === expected[k]) k++;
console.log(` first divergence at byte ${k}: got ${JSON.stringify(r.stdout.slice(k, k + 40))} want ${JSON.stringify(expected.slice(k, k + 40))}`);
}
}
} finally {
await sandbox.kill().catch(() => {});
}

// ---------- 6. Cold create time ----------
console.log("\n== 6. Cold sandbox creates (default desktop template) ==");
const createTimes: number[] = [];
for (let i = 0; i < 3; i++) {
const t0 = performance.now();
const sb = await E2BSandbox.create({ timeoutMs: 120_000 }).catch((e) => {
console.log(` create ${i} failed: ${(e as Error).message}`);
return null;
});
const ms = performance.now() - t0;
if (!sb) continue;
createTimes.push(ms);
await sb.kill().catch(() => {});
}
report("Sandbox.create cold", createTimes);
}

await main();
Loading