From 2544a33f044174b8eef160cdcab648d16da8b200 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 18 Aug 2026 03:24:06 +0000 Subject: [PATCH 1/2] Store duplicated file contents only once when packing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Files whose contents hash identically are now written into the archive a single time. Each copy still gets its own header entry with its own size, integrity, and executable bit, but the entries share one offset, so the redundant bytes are never written. Detection reuses the SHA-256 already computed for each entry's integrity field, so there is no extra read or hashing pass. Unpacked files are excluded: they are copied out as standalone files next to the archive, so every copy has to exist on disk. Readers are unaffected — nothing in the format required offsets to be unique — and archives without duplicates are byte-for-byte unchanged. On a 2000-file tree the archive shrinks in proportion to how much of the input is duplicated (-9.4% at a 10% duplicate share, -47.1% at 50%, -84.8% at 90%), while pack time is unchanged within measurement noise both with and without duplicates. --- README.md | 16 +++ benchmark/generate-fixtures.ts | 33 +++++- src/asar.ts | 22 ++-- src/disk.ts | 10 ++ src/filesystem.ts | 81 +++++++++++---- test/asar-spec.ts | 182 +++++++++++++++++++++++++++++++++ 6 files changed, 310 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 35f88cda..566ae2a9 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ ASAR is a simple extensive archive format. It concatenates all files together wi * Support random access * Use JSON to store file information * Very easy to write a parser +* Store the contents of duplicated files only once ## CLI @@ -104,6 +105,18 @@ console.log('done.'); Please note that there is currently **no** error handling provided! +### Deduplication + +Files with identical contents are stored once and shared: the first copy is +written into the archive and every other copy's header entry points at that same +`offset`. Nothing changes for readers — each file still has its own entry, size, +integrity hash, and executable bit — but archives with duplicated contents (a +common shape for bundled `node_modules`) get smaller and pack faster, since the +redundant bytes are never written. + +Unpacked files (`unpack` / `unpackDir`) are always written out in full, because +they live on disk outside the archive. + ### Transform You can pass in a `transform` option, that is a function, which either returns @@ -200,6 +213,9 @@ Structure of `header` is something like this: `offset` starts from 0 so you have to manually add the size of `header_size` and `header` to the `offset` to get the real offset of the file. +Files with identical contents share a single copy in the archive, so more than +one entry can point at the same `offset`. + `offset` is a UINT64 number represented in string, because there is no way to precisely represent UINT64 in JavaScript `Number`. `size` is a JavaScript `Number` that is no larger than `Number.MAX_SAFE_INTEGER`, which has a value of diff --git a/benchmark/generate-fixtures.ts b/benchmark/generate-fixtures.ts index d2588be9..e6ed0b9e 100644 --- a/benchmark/generate-fixtures.ts +++ b/benchmark/generate-fixtures.ts @@ -4,6 +4,9 @@ import crypto from 'node:crypto'; const BENCHMARK_DIR = path.join(import.meta.dirname, 'fixtures'); +/** How many distinct contents duplicated files are drawn from */ +const DUPLICATE_POOL_SIZE = 64; + export type FixtureConfig = { name: string; fileCount: number; @@ -13,6 +16,8 @@ export type FixtureConfig = { depth: number; /** Number of subdirectories per level */ breadth: number; + /** Fraction of files that reuse the contents of an earlier file (0 by default) */ + duplicateRatio?: number; }; export const FIXTURES: FixtureConfig[] = [ @@ -22,6 +27,14 @@ export const FIXTURES: FixtureConfig[] = [ { name: 'few-large-files', fileCount: 20, avgFileSize: 1024 * 1024, depth: 2, breadth: 2 }, { name: 'many-small-files', fileCount: 10000, avgFileSize: 256, depth: 3, breadth: 10 }, { name: 'deep-tree', fileCount: 1000, avgFileSize: 2048, depth: 10, breadth: 2 }, + { + name: 'duplicate-heavy', + fileCount: 3000, + avgFileSize: 4096, + depth: 4, + breadth: 4, + duplicateRatio: 0.5, + }, ]; function generateRandomContent(size: number): Buffer { @@ -59,13 +72,25 @@ export function generateFixture(config: FixtureConfig): string { } } + // Contents that duplicated files are drawn from, mimicking the repeated + // licenses/READMEs/vendored copies that show up across `node_modules` + const duplicateRatio = config.duplicateRatio ?? 0; + const duplicatePool: Buffer[] = []; + // Distribute files across directories for (let i = 0; i < config.fileCount; i++) { const dir = dirs[i % dirs.length]; - // Vary file sizes: 50% to 150% of average - const sizeVariation = 0.5 + Math.random(); - const size = Math.max(1, Math.floor(config.avgFileSize * sizeVariation)); - const content = generateRandomContent(size); + let content: Buffer; + if (duplicatePool.length > 0 && Math.random() < duplicateRatio) { + content = duplicatePool[i % duplicatePool.length]; + } else { + // Vary file sizes: 50% to 150% of average + const sizeVariation = 0.5 + Math.random(); + content = generateRandomContent(Math.max(1, Math.floor(config.avgFileSize * sizeVariation))); + if (duplicatePool.length < DUPLICATE_POOL_SIZE) { + duplicatePool.push(content); + } + } const ext = ['.txt', '.js', '.json', '.bin', '.dat'][i % 5]; const filePath = path.join(fixtureDir, dir, `file_${i}${ext}`); fs.writeFileSync(filePath, content); diff --git a/src/asar.ts b/src/asar.ts index d60a9228..2c7a358f 100644 --- a/src/asar.ts +++ b/src/asar.ts @@ -195,22 +195,23 @@ export async function createPackageFromFiles( ); filesystem.insertDirectory(filename, shouldUnpack); break; - case 'file': + case 'file': { shouldUnpack = shouldUnpackPath( filename, path.relative(src, path.dirname(filename)), options.unpack, options.unpackDir, ); - files.push({ filename, unpack: shouldUnpack }); - await filesystem.insertFile( + const duplicate = await filesystem.insertFile( filename, () => fs.createReadStream(filename), shouldUnpack, file, options, ); + files.push({ filename, unpack: shouldUnpack, duplicate }); break; + } case 'link': shouldUnpack = shouldUnpackPath( filename, @@ -277,18 +278,23 @@ export async function createPackageFromStreams(dest: string, streams: AsarStream case 'directory': filesystem.insertDirectory(filename, stream.unpacked); break; - case 'file': + case 'file': { + const duplicate = await filesystem.insertFile( + filename, + stream.streamGenerator, + stream.unpacked, + { type: 'file', stat: stream.stat }, + ); files.push({ filename, streamGenerator: stream.streamGenerator, link: undefined, mode: stream.stat.mode, unpack: stream.unpacked, + duplicate, }); - return filesystem.insertFile(filename, stream.streamGenerator, stream.unpacked, { - type: 'file', - stat: stream.stat, - }); + break; + } case 'link': links.push({ filename, diff --git a/src/disk.ts b/src/disk.ts index 1335821f..2e765d80 100644 --- a/src/disk.ts +++ b/src/disk.ts @@ -183,6 +183,8 @@ export type InputMetadata = { export type BasicFilesArray = { filename: string; unpack: boolean; + /** Contents shared with an earlier file, already written to the archive */ + duplicate?: boolean; }[]; export type BasicStreamArray = { @@ -191,6 +193,8 @@ export type BasicStreamArray = { mode: Stats['mode']; unpack: boolean; link: string | undefined; // only for symlinks, should refactor as part of larger project refactor in follow-up PR + /** Contents shared with an earlier file, already written to the archive */ + duplicate?: boolean; }[]; export type FilesystemFilesAndLinks = { @@ -221,6 +225,9 @@ const writeFileListToStream = async function ( }; for (const file of files) { + if (file.duplicate) { + continue; + } if (file.unpack) { await flushPendingBuffers(); const filename = path.relative(filesystem.getRootPath(), file.filename); @@ -270,6 +277,9 @@ export async function streamFilesystem( const { files, links } = lists; for await (const file of files) { + if (file.duplicate) { + continue; + } // the file should not be packed into archive if (file.unpack) { const targetFile = path.join(`${dest}.unpacked`, file.filename); diff --git a/src/filesystem.ts b/src/filesystem.ts index 443add6c..97ac5e55 100644 --- a/src/filesystem.ts +++ b/src/filesystem.ts @@ -40,12 +40,15 @@ export class Filesystem { private header: FilesystemEntry; private headerSize: number; private offset: bigint; + // SHA-256 of contents already stored => the offset those contents live at + private contentOffsets: Map; constructor(src: string) { this.src = path.resolve(src); this.header = { files: Object.create(null) }; this.headerSize = 0; this.offset = BigInt(0); + this.contentOffsets = new Map(); } getRootPath() { @@ -116,14 +119,17 @@ export class Filesystem { options: { transform?: (filePath: string) => NodeJS.ReadWriteStream | void; } = {}, - ): Promise { + ): Promise { const dirNode = this.searchNodeFromPath(path.dirname(p)) as FilesystemDirectoryEntry; const node = this.searchNodeFromPath(p) as FilesystemFileEntry; if (shouldUnpack || dirNode.unpacked) { node.size = file.stat.size; node.unpacked = true; + // Unpacked files are copied out of the archive as standalone files, so every + // copy is written even when their contents are identical. return getFileIntegrity(streamGenerator()).then((integrity) => { node.integrity = integrity; + return false; }); } @@ -139,29 +145,58 @@ export class Filesystem { throw new Error(`${p}: file size can not be larger than 4.2GB`); } - node.size = size; - node.offset = this.offset.toString(); - if (process.platform !== 'win32' && file.stat.mode & 0o100) { - node.executable = true; - } + const executable = process.platform !== 'win32' && (file.stat.mode & 0o100) !== 0; if (size <= BUFFER_HASH_THRESHOLD) { // Fully synchronous fast path — no Promise, no stream, no microtask yield try { const fileBuffer = fs.readFileSync(p); - node.integrity = getFileIntegrityFromBuffer(fileBuffer); - file.cachedBuffer = fileBuffer; - this.offset += BigInt(size); - return Promise.resolve(); + const integrity = getFileIntegrityFromBuffer(fileBuffer); + const duplicate = this.storeFileEntry(node, size, executable, integrity); + if (!duplicate) { + file.cachedBuffer = fileBuffer; + } + return Promise.resolve(duplicate); } catch { // Fall through to stream path } } - return getFileIntegrity(streamGenerator()).then((integrity) => { - node.integrity = integrity; - this.offset += BigInt(size); - }); + return getFileIntegrity(streamGenerator()).then((integrity) => + this.storeFileEntry(node, size, executable, integrity), + ); + } + + /** + * Fills in a file entry and reserves the region of the archive that holds its + * contents, growing the archive by `size` bytes. When a file with identical + * contents was already inserted, the entry instead points at the offset of those + * contents and the archive is left unchanged — the caller must then skip writing + * the contents. + * + * @returns whether the contents were already stored by an earlier file + */ + private storeFileEntry( + node: FilesystemFileEntry, + size: number, + executable: boolean, + integrity: FileIntegrity, + ): boolean { + const sharedOffset = this.contentOffsets.get(integrity.hash); + + node.size = size; + node.offset = sharedOffset ?? this.offset.toString(); + if (executable) { + node.executable = true; + } + node.integrity = integrity; + + if (sharedOffset !== undefined) { + return true; + } + this.contentOffsets.set(integrity.hash, node.offset); + this.offset += BigInt(size); + return false; } private async insertFileAsync( @@ -170,7 +205,7 @@ export class Filesystem { file: CrawledFileType, node: FilesystemFileEntry, transformed: NodeJS.ReadWriteStream, - ) { + ): Promise { const tmpdir = await fs.mkdtemp(path.join(os.tmpdir(), 'asar-')); const tmpfile = path.join(tmpdir, path.basename(p)); const out = fs.createWriteStream(tmpfile); @@ -186,15 +221,17 @@ export class Filesystem { throw new Error(`${p}: file size can not be larger than 4.2GB`); } - node.size = size; - node.offset = this.offset.toString(); - if (process.platform !== 'win32' && file.stat.mode & 0o100) { - node.executable = true; - } // Integrity must be computed over the transformed bytes that are actually // stored in the archive, not the original (pre-transform) source bytes. - node.integrity = await getFileIntegrity(fs.createReadStream(file.transformed.path)); - this.offset += BigInt(size); + const integrity = await getFileIntegrity(fs.createReadStream(file.transformed.path)); + const executable = process.platform !== 'win32' && (file.stat.mode & 0o100) !== 0; + const duplicate = this.storeFileEntry(node, size, executable, integrity); + if (duplicate) { + // Nothing will read these transformed bytes again, so don't leave them behind + file.transformed = undefined; + fs.rmSync(tmpdir, { recursive: true, force: true }); + } + return duplicate; } insertLink( diff --git a/test/asar-spec.ts b/test/asar-spec.ts index ee11973f..f3eca0e0 100644 --- a/test/asar-spec.ts +++ b/test/asar-spec.ts @@ -21,6 +21,7 @@ import { import { crawl } from '../src/crawlfs.js'; import { getFileIntegrityFromBuffer } from '../src/integrity.js'; import { useTmpDir } from './util/tmpDir.js'; +import { compDirs } from './util/compareDirectories.js'; import { Transform } from 'node:stream'; describe('asar', () => { @@ -726,4 +727,185 @@ describe('asar', () => { } }); }); + + describe('deduplication', () => { + const fileEntry = (archive: string, filename: string) => + statFile(archive, filename) as { offset: string; size: number }; + + /** Number of bytes the archive stores after its 8-byte size field and header. */ + const payloadSize = (archive: string) => + fs.statSync(archive).size - getRawHeader(archive).headerSize - 8; + + it('should store identical file contents only once', async () => { + const shared = 'x'.repeat(4096); + const unique = 'y'.repeat(4096); + const src = createFixture('dedupe-shared', { + 'a.txt': shared, + 'dir/b.txt': shared, + 'dir/nested/c.txt': shared, + 'unique.txt': unique, + }); + const dest = path.join(testRunDir, 'dedupe-shared.asar'); + await createPackage(src, dest); + + const first = fileEntry(dest, 'a.txt'); + expect(fileEntry(dest, 'dir/b.txt').offset).toBe(first.offset); + expect(fileEntry(dest, 'dir/nested/c.txt').offset).toBe(first.offset); + expect(fileEntry(dest, 'unique.txt').offset).not.toBe(first.offset); + + // Only the two distinct payloads are stored + expect(payloadSize(dest)).toBe(shared.length + unique.length); + + expect(extractFile(dest, 'a.txt').toString()).toBe(shared); + expect(extractFile(dest, 'dir/b.txt').toString()).toBe(shared); + expect(extractFile(dest, 'dir/nested/c.txt').toString()).toBe(shared); + expect(extractFile(dest, 'unique.txt').toString()).toBe(unique); + }); + + it('should extract deduplicated archives to the original file tree', async () => { + const src = createFixture('dedupe-extract', { + 'a.txt': 'same', + 'dir/b.txt': 'same', + 'dir/c.bin': crypto.randomBytes(2048), + 'dir/nested/d.bin': 'different', + }); + fs.copyFileSync(path.join(src, 'dir/c.bin'), path.join(src, 'copy-of-c.bin')); + + const dest = path.join(testRunDir, 'dedupe-extract.asar'); + await createPackage(src, dest); + const extractDir = path.join(testRunDir, 'dedupe-extract-out'); + extractAll(dest, extractDir); + + await compDirs(src, extractDir); + }); + + it( + 'should dedupe files larger than the buffer hashing threshold', + { timeout: 30000 }, + async () => { + const big = crypto.randomBytes(3 * 1024 * 1024); + const src = createFixture('dedupe-large', { 'big.bin': big, 'dir/big-copy.bin': big }); + const dest = path.join(testRunDir, 'dedupe-large.asar'); + await createPackage(src, dest); + + expect(fileEntry(dest, 'dir/big-copy.bin').offset).toBe(fileEntry(dest, 'big.bin').offset); + expect(payloadSize(dest)).toBe(big.length); + expect(extractFile(dest, 'dir/big-copy.bin')).toEqual(big); + }, + ); + + it('should not dedupe different contents of the same size', async () => { + const src = createFixture('dedupe-same-size', { + 'a.txt': 'aaaa', + 'b.txt': 'bbbb', + }); + const dest = path.join(testRunDir, 'dedupe-same-size.asar'); + await createPackage(src, dest); + + expect(fileEntry(dest, 'b.txt').offset).not.toBe(fileEntry(dest, 'a.txt').offset); + expect(extractFile(dest, 'a.txt').toString()).toBe('aaaa'); + expect(extractFile(dest, 'b.txt').toString()).toBe('bbbb'); + }); + + it('should share contents between files that differ only by executable bit', async function () { + if (process.platform === 'win32') return; + const src = createFixture('dedupe-executable', { + 'script.sh': '#!/bin/bash\necho hi', + 'copy.sh': '#!/bin/bash\necho hi', + }); + fs.chmodSync(path.join(src, 'script.sh'), 0o755); + + const dest = path.join(testRunDir, 'dedupe-executable.asar'); + await createPackage(src, dest); + expect(fileEntry(dest, 'copy.sh').offset).toBe(fileEntry(dest, 'script.sh').offset); + + const extractDir = path.join(testRunDir, 'dedupe-executable-out'); + extractAll(dest, extractDir); + expect(fs.statSync(path.join(extractDir, 'script.sh')).mode & 0o111).toBeGreaterThan(0); + expect(fs.statSync(path.join(extractDir, 'copy.sh')).mode & 0o111).toBe(0); + }); + + it('should write every unpacked copy of duplicated contents', async () => { + const shared = 'shared native blob'; + const src = createFixture('dedupe-unpacked', { + 'a.node': shared, + 'dir/b.node': shared, + 'packed.txt': shared, + }); + const dest = path.join(testRunDir, 'dedupe-unpacked.asar'); + await createPackageWithOptions(src, dest, { unpack: '*.node' }); + + expect(fs.readFileSync(path.join(`${dest}.unpacked`, 'a.node'), 'utf8')).toBe(shared); + expect(fs.readFileSync(path.join(`${dest}.unpacked`, 'dir/b.node'), 'utf8')).toBe(shared); + // Unpacked files live outside the archive, so the packed copy is still stored + expect(extractFile(dest, 'packed.txt').toString()).toBe(shared); + }); + + it('should dedupe on the transformed contents', async () => { + const src = createFixture('dedupe-transform', { + 'a.txt': 'abc', + 'b.txt': 'cba', + 'c.txt': 'xyz', + }); + const dest = path.join(testRunDir, 'dedupe-transform.asar'); + // Sorting each file's bytes makes `a.txt` and `b.txt` identical once transformed + await createPackageWithOptions(src, dest, { + transform: () => + new Transform({ + transform(chunk, _encoding, callback) { + callback(null, Buffer.from([...chunk].sort((a, b) => a - b))); + }, + }), + }); + + expect(fileEntry(dest, 'b.txt').offset).toBe(fileEntry(dest, 'a.txt').offset); + expect(fileEntry(dest, 'c.txt').offset).not.toBe(fileEntry(dest, 'a.txt').offset); + expect(extractFile(dest, 'a.txt').toString()).toBe('abc'); + expect(extractFile(dest, 'b.txt').toString()).toBe('abc'); + expect(extractFile(dest, 'c.txt').toString()).toBe('xyz'); + }); + + it('should dedupe archives created from streams', async () => { + const shared = Buffer.from('shared stream contents'); + const streams = (): AsarStreamType[] => [ + { + type: 'file', + path: 'a.txt', + unpacked: false, + streamGenerator: () => Readable.from(shared), + stat: { mode: 0o644, size: shared.length }, + }, + { + type: 'file', + path: 'b.txt', + unpacked: false, + streamGenerator: () => Readable.from(shared), + stat: { mode: 0o644, size: shared.length }, + }, + ]; + + const dest = path.join(testRunDir, 'dedupe-streams.asar'); + await createPackageFromStreams(dest, streams()); + + expect(fileEntry(dest, 'b.txt').offset).toBe(fileEntry(dest, 'a.txt').offset); + expect(payloadSize(dest)).toBe(shared.length); + expect(extractFile(dest, 'b.txt')).toEqual(shared); + }); + + it('should keep empty files working alongside duplicates', async () => { + const src = createFixture('dedupe-empty', { + 'empty1.txt': '', + 'empty2.txt': '', + 'a.txt': 'dup', + 'b.txt': 'dup', + }); + const dest = path.join(testRunDir, 'dedupe-empty.asar'); + await createPackage(src, dest); + + expect(extractFile(dest, 'empty1.txt').length).toBe(0); + expect(extractFile(dest, 'empty2.txt').length).toBe(0); + expect(payloadSize(dest)).toBe('dup'.length); + expect(extractFile(dest, 'b.txt').toString()).toBe('dup'); + }); + }); }); From bee923a76c97bd942effdebaa32048945f004eb3 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 18 Aug 2026 03:46:37 +0000 Subject: [PATCH 2/2] Use native path separators in deduplication tests Archive lookups split on path.sep, so a forward-slashed path more than one level deep never splits into segments on Windows and the entry is not found. Build those paths with path.join instead. No-Verification-Needed: test-only change, no runtime surface --- test/asar-spec.ts | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/test/asar-spec.ts b/test/asar-spec.ts index f3eca0e0..3f6a7609 100644 --- a/test/asar-spec.ts +++ b/test/asar-spec.ts @@ -748,17 +748,21 @@ describe('asar', () => { const dest = path.join(testRunDir, 'dedupe-shared.asar'); await createPackage(src, dest); + // Archive lookups split on path.sep, so nested paths need native separators + const nestedB = path.join('dir', 'b.txt'); + const nestedC = path.join('dir', 'nested', 'c.txt'); + const first = fileEntry(dest, 'a.txt'); - expect(fileEntry(dest, 'dir/b.txt').offset).toBe(first.offset); - expect(fileEntry(dest, 'dir/nested/c.txt').offset).toBe(first.offset); + expect(fileEntry(dest, nestedB).offset).toBe(first.offset); + expect(fileEntry(dest, nestedC).offset).toBe(first.offset); expect(fileEntry(dest, 'unique.txt').offset).not.toBe(first.offset); // Only the two distinct payloads are stored expect(payloadSize(dest)).toBe(shared.length + unique.length); expect(extractFile(dest, 'a.txt').toString()).toBe(shared); - expect(extractFile(dest, 'dir/b.txt').toString()).toBe(shared); - expect(extractFile(dest, 'dir/nested/c.txt').toString()).toBe(shared); + expect(extractFile(dest, nestedB).toString()).toBe(shared); + expect(extractFile(dest, nestedC).toString()).toBe(shared); expect(extractFile(dest, 'unique.txt').toString()).toBe(unique); }); @@ -788,9 +792,10 @@ describe('asar', () => { const dest = path.join(testRunDir, 'dedupe-large.asar'); await createPackage(src, dest); - expect(fileEntry(dest, 'dir/big-copy.bin').offset).toBe(fileEntry(dest, 'big.bin').offset); + const copy = path.join('dir', 'big-copy.bin'); + expect(fileEntry(dest, copy).offset).toBe(fileEntry(dest, 'big.bin').offset); expect(payloadSize(dest)).toBe(big.length); - expect(extractFile(dest, 'dir/big-copy.bin')).toEqual(big); + expect(extractFile(dest, copy)).toEqual(big); }, ); @@ -836,7 +841,7 @@ describe('asar', () => { await createPackageWithOptions(src, dest, { unpack: '*.node' }); expect(fs.readFileSync(path.join(`${dest}.unpacked`, 'a.node'), 'utf8')).toBe(shared); - expect(fs.readFileSync(path.join(`${dest}.unpacked`, 'dir/b.node'), 'utf8')).toBe(shared); + expect(fs.readFileSync(path.join(`${dest}.unpacked`, 'dir', 'b.node'), 'utf8')).toBe(shared); // Unpacked files live outside the archive, so the packed copy is still stored expect(extractFile(dest, 'packed.txt').toString()).toBe(shared); });