diff --git a/README.md b/README.md index 35f88cda..566ae2a9 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ ASAR is a simple extensive archive format. It concatenates all files together wi * Support random access * Use JSON to store file information * Very easy to write a parser +* Store the contents of duplicated files only once ## CLI @@ -104,6 +105,18 @@ console.log('done.'); Please note that there is currently **no** error handling provided! +### Deduplication + +Files with identical contents are stored once and shared: the first copy is +written into the archive and every other copy's header entry points at that same +`offset`. Nothing changes for readers — each file still has its own entry, size, +integrity hash, and executable bit — but archives with duplicated contents (a +common shape for bundled `node_modules`) get smaller and pack faster, since the +redundant bytes are never written. + +Unpacked files (`unpack` / `unpackDir`) are always written out in full, because +they live on disk outside the archive. + ### Transform You can pass in a `transform` option, that is a function, which either returns @@ -200,6 +213,9 @@ Structure of `header` is something like this: `offset` starts from 0 so you have to manually add the size of `header_size` and `header` to the `offset` to get the real offset of the file. +Files with identical contents share a single copy in the archive, so more than +one entry can point at the same `offset`. + `offset` is a UINT64 number represented in string, because there is no way to precisely represent UINT64 in JavaScript `Number`. `size` is a JavaScript `Number` that is no larger than `Number.MAX_SAFE_INTEGER`, which has a value of diff --git a/benchmark/generate-fixtures.ts b/benchmark/generate-fixtures.ts index d2588be9..e6ed0b9e 100644 --- a/benchmark/generate-fixtures.ts +++ b/benchmark/generate-fixtures.ts @@ -4,6 +4,9 @@ import crypto from 'node:crypto'; const BENCHMARK_DIR = path.join(import.meta.dirname, 'fixtures'); +/** How many distinct contents duplicated files are drawn from */ +const DUPLICATE_POOL_SIZE = 64; + export type FixtureConfig = { name: string; fileCount: number; @@ -13,6 +16,8 @@ export type FixtureConfig = { depth: number; /** Number of subdirectories per level */ breadth: number; + /** Fraction of files that reuse the contents of an earlier file (0 by default) */ + duplicateRatio?: number; }; export const FIXTURES: FixtureConfig[] = [ @@ -22,6 +27,14 @@ export const FIXTURES: FixtureConfig[] = [ { name: 'few-large-files', fileCount: 20, avgFileSize: 1024 * 1024, depth: 2, breadth: 2 }, { name: 'many-small-files', fileCount: 10000, avgFileSize: 256, depth: 3, breadth: 10 }, { name: 'deep-tree', fileCount: 1000, avgFileSize: 2048, depth: 10, breadth: 2 }, + { + name: 'duplicate-heavy', + fileCount: 3000, + avgFileSize: 4096, + depth: 4, + breadth: 4, + duplicateRatio: 0.5, + }, ]; function generateRandomContent(size: number): Buffer { @@ -59,13 +72,25 @@ export function generateFixture(config: FixtureConfig): string { } } + // Contents that duplicated files are drawn from, mimicking the repeated + // licenses/READMEs/vendored copies that show up across `node_modules` + const duplicateRatio = config.duplicateRatio ?? 0; + const duplicatePool: Buffer[] = []; + // Distribute files across directories for (let i = 0; i < config.fileCount; i++) { const dir = dirs[i % dirs.length]; - // Vary file sizes: 50% to 150% of average - const sizeVariation = 0.5 + Math.random(); - const size = Math.max(1, Math.floor(config.avgFileSize * sizeVariation)); - const content = generateRandomContent(size); + let content: Buffer; + if (duplicatePool.length > 0 && Math.random() < duplicateRatio) { + content = duplicatePool[i % duplicatePool.length]; + } else { + // Vary file sizes: 50% to 150% of average + const sizeVariation = 0.5 + Math.random(); + content = generateRandomContent(Math.max(1, Math.floor(config.avgFileSize * sizeVariation))); + if (duplicatePool.length < DUPLICATE_POOL_SIZE) { + duplicatePool.push(content); + } + } const ext = ['.txt', '.js', '.json', '.bin', '.dat'][i % 5]; const filePath = path.join(fixtureDir, dir, `file_${i}${ext}`); fs.writeFileSync(filePath, content); diff --git a/src/asar.ts b/src/asar.ts index d60a9228..2c7a358f 100644 --- a/src/asar.ts +++ b/src/asar.ts @@ -195,22 +195,23 @@ export async function createPackageFromFiles( ); filesystem.insertDirectory(filename, shouldUnpack); break; - case 'file': + case 'file': { shouldUnpack = shouldUnpackPath( filename, path.relative(src, path.dirname(filename)), options.unpack, options.unpackDir, ); - files.push({ filename, unpack: shouldUnpack }); - await filesystem.insertFile( + const duplicate = await filesystem.insertFile( filename, () => fs.createReadStream(filename), shouldUnpack, file, options, ); + files.push({ filename, unpack: shouldUnpack, duplicate }); break; + } case 'link': shouldUnpack = shouldUnpackPath( filename, @@ -277,18 +278,23 @@ export async function createPackageFromStreams(dest: string, streams: AsarStream case 'directory': filesystem.insertDirectory(filename, stream.unpacked); break; - case 'file': + case 'file': { + const duplicate = await filesystem.insertFile( + filename, + stream.streamGenerator, + stream.unpacked, + { type: 'file', stat: stream.stat }, + ); files.push({ filename, streamGenerator: stream.streamGenerator, link: undefined, mode: stream.stat.mode, unpack: stream.unpacked, + duplicate, }); - return filesystem.insertFile(filename, stream.streamGenerator, stream.unpacked, { - type: 'file', - stat: stream.stat, - }); + break; + } case 'link': links.push({ filename, diff --git a/src/disk.ts b/src/disk.ts index 1335821f..2e765d80 100644 --- a/src/disk.ts +++ b/src/disk.ts @@ -183,6 +183,8 @@ export type InputMetadata = { export type BasicFilesArray = { filename: string; unpack: boolean; + /** Contents shared with an earlier file, already written to the archive */ + duplicate?: boolean; }[]; export type BasicStreamArray = { @@ -191,6 +193,8 @@ export type BasicStreamArray = { mode: Stats['mode']; unpack: boolean; link: string | undefined; // only for symlinks, should refactor as part of larger project refactor in follow-up PR + /** Contents shared with an earlier file, already written to the archive */ + duplicate?: boolean; }[]; export type FilesystemFilesAndLinks = { @@ -221,6 +225,9 @@ const writeFileListToStream = async function ( }; for (const file of files) { + if (file.duplicate) { + continue; + } if (file.unpack) { await flushPendingBuffers(); const filename = path.relative(filesystem.getRootPath(), file.filename); @@ -270,6 +277,9 @@ export async function streamFilesystem( const { files, links } = lists; for await (const file of files) { + if (file.duplicate) { + continue; + } // the file should not be packed into archive if (file.unpack) { const targetFile = path.join(`${dest}.unpacked`, file.filename); diff --git a/src/filesystem.ts b/src/filesystem.ts index 443add6c..97ac5e55 100644 --- a/src/filesystem.ts +++ b/src/filesystem.ts @@ -40,12 +40,15 @@ export class Filesystem { private header: FilesystemEntry; private headerSize: number; private offset: bigint; + // SHA-256 of contents already stored => the offset those contents live at + private contentOffsets: Map; constructor(src: string) { this.src = path.resolve(src); this.header = { files: Object.create(null) }; this.headerSize = 0; this.offset = BigInt(0); + this.contentOffsets = new Map(); } getRootPath() { @@ -116,14 +119,17 @@ export class Filesystem { options: { transform?: (filePath: string) => NodeJS.ReadWriteStream | void; } = {}, - ): Promise { + ): Promise { const dirNode = this.searchNodeFromPath(path.dirname(p)) as FilesystemDirectoryEntry; const node = this.searchNodeFromPath(p) as FilesystemFileEntry; if (shouldUnpack || dirNode.unpacked) { node.size = file.stat.size; node.unpacked = true; + // Unpacked files are copied out of the archive as standalone files, so every + // copy is written even when their contents are identical. return getFileIntegrity(streamGenerator()).then((integrity) => { node.integrity = integrity; + return false; }); } @@ -139,29 +145,58 @@ export class Filesystem { throw new Error(`${p}: file size can not be larger than 4.2GB`); } - node.size = size; - node.offset = this.offset.toString(); - if (process.platform !== 'win32' && file.stat.mode & 0o100) { - node.executable = true; - } + const executable = process.platform !== 'win32' && (file.stat.mode & 0o100) !== 0; if (size <= BUFFER_HASH_THRESHOLD) { // Fully synchronous fast path — no Promise, no stream, no microtask yield try { const fileBuffer = fs.readFileSync(p); - node.integrity = getFileIntegrityFromBuffer(fileBuffer); - file.cachedBuffer = fileBuffer; - this.offset += BigInt(size); - return Promise.resolve(); + const integrity = getFileIntegrityFromBuffer(fileBuffer); + const duplicate = this.storeFileEntry(node, size, executable, integrity); + if (!duplicate) { + file.cachedBuffer = fileBuffer; + } + return Promise.resolve(duplicate); } catch { // Fall through to stream path } } - return getFileIntegrity(streamGenerator()).then((integrity) => { - node.integrity = integrity; - this.offset += BigInt(size); - }); + return getFileIntegrity(streamGenerator()).then((integrity) => + this.storeFileEntry(node, size, executable, integrity), + ); + } + + /** + * Fills in a file entry and reserves the region of the archive that holds its + * contents, growing the archive by `size` bytes. When a file with identical + * contents was already inserted, the entry instead points at the offset of those + * contents and the archive is left unchanged — the caller must then skip writing + * the contents. + * + * @returns whether the contents were already stored by an earlier file + */ + private storeFileEntry( + node: FilesystemFileEntry, + size: number, + executable: boolean, + integrity: FileIntegrity, + ): boolean { + const sharedOffset = this.contentOffsets.get(integrity.hash); + + node.size = size; + node.offset = sharedOffset ?? this.offset.toString(); + if (executable) { + node.executable = true; + } + node.integrity = integrity; + + if (sharedOffset !== undefined) { + return true; + } + this.contentOffsets.set(integrity.hash, node.offset); + this.offset += BigInt(size); + return false; } private async insertFileAsync( @@ -170,7 +205,7 @@ export class Filesystem { file: CrawledFileType, node: FilesystemFileEntry, transformed: NodeJS.ReadWriteStream, - ) { + ): Promise { const tmpdir = await fs.mkdtemp(path.join(os.tmpdir(), 'asar-')); const tmpfile = path.join(tmpdir, path.basename(p)); const out = fs.createWriteStream(tmpfile); @@ -186,15 +221,17 @@ export class Filesystem { throw new Error(`${p}: file size can not be larger than 4.2GB`); } - node.size = size; - node.offset = this.offset.toString(); - if (process.platform !== 'win32' && file.stat.mode & 0o100) { - node.executable = true; - } // Integrity must be computed over the transformed bytes that are actually // stored in the archive, not the original (pre-transform) source bytes. - node.integrity = await getFileIntegrity(fs.createReadStream(file.transformed.path)); - this.offset += BigInt(size); + const integrity = await getFileIntegrity(fs.createReadStream(file.transformed.path)); + const executable = process.platform !== 'win32' && (file.stat.mode & 0o100) !== 0; + const duplicate = this.storeFileEntry(node, size, executable, integrity); + if (duplicate) { + // Nothing will read these transformed bytes again, so don't leave them behind + file.transformed = undefined; + fs.rmSync(tmpdir, { recursive: true, force: true }); + } + return duplicate; } insertLink( diff --git a/test/asar-spec.ts b/test/asar-spec.ts index ee11973f..3f6a7609 100644 --- a/test/asar-spec.ts +++ b/test/asar-spec.ts @@ -21,6 +21,7 @@ import { import { crawl } from '../src/crawlfs.js'; import { getFileIntegrityFromBuffer } from '../src/integrity.js'; import { useTmpDir } from './util/tmpDir.js'; +import { compDirs } from './util/compareDirectories.js'; import { Transform } from 'node:stream'; describe('asar', () => { @@ -726,4 +727,190 @@ describe('asar', () => { } }); }); + + describe('deduplication', () => { + const fileEntry = (archive: string, filename: string) => + statFile(archive, filename) as { offset: string; size: number }; + + /** Number of bytes the archive stores after its 8-byte size field and header. */ + const payloadSize = (archive: string) => + fs.statSync(archive).size - getRawHeader(archive).headerSize - 8; + + it('should store identical file contents only once', async () => { + const shared = 'x'.repeat(4096); + const unique = 'y'.repeat(4096); + const src = createFixture('dedupe-shared', { + 'a.txt': shared, + 'dir/b.txt': shared, + 'dir/nested/c.txt': shared, + 'unique.txt': unique, + }); + const dest = path.join(testRunDir, 'dedupe-shared.asar'); + await createPackage(src, dest); + + // Archive lookups split on path.sep, so nested paths need native separators + const nestedB = path.join('dir', 'b.txt'); + const nestedC = path.join('dir', 'nested', 'c.txt'); + + const first = fileEntry(dest, 'a.txt'); + expect(fileEntry(dest, nestedB).offset).toBe(first.offset); + expect(fileEntry(dest, nestedC).offset).toBe(first.offset); + expect(fileEntry(dest, 'unique.txt').offset).not.toBe(first.offset); + + // Only the two distinct payloads are stored + expect(payloadSize(dest)).toBe(shared.length + unique.length); + + expect(extractFile(dest, 'a.txt').toString()).toBe(shared); + expect(extractFile(dest, nestedB).toString()).toBe(shared); + expect(extractFile(dest, nestedC).toString()).toBe(shared); + expect(extractFile(dest, 'unique.txt').toString()).toBe(unique); + }); + + it('should extract deduplicated archives to the original file tree', async () => { + const src = createFixture('dedupe-extract', { + 'a.txt': 'same', + 'dir/b.txt': 'same', + 'dir/c.bin': crypto.randomBytes(2048), + 'dir/nested/d.bin': 'different', + }); + fs.copyFileSync(path.join(src, 'dir/c.bin'), path.join(src, 'copy-of-c.bin')); + + const dest = path.join(testRunDir, 'dedupe-extract.asar'); + await createPackage(src, dest); + const extractDir = path.join(testRunDir, 'dedupe-extract-out'); + extractAll(dest, extractDir); + + await compDirs(src, extractDir); + }); + + it( + 'should dedupe files larger than the buffer hashing threshold', + { timeout: 30000 }, + async () => { + const big = crypto.randomBytes(3 * 1024 * 1024); + const src = createFixture('dedupe-large', { 'big.bin': big, 'dir/big-copy.bin': big }); + const dest = path.join(testRunDir, 'dedupe-large.asar'); + await createPackage(src, dest); + + const copy = path.join('dir', 'big-copy.bin'); + expect(fileEntry(dest, copy).offset).toBe(fileEntry(dest, 'big.bin').offset); + expect(payloadSize(dest)).toBe(big.length); + expect(extractFile(dest, copy)).toEqual(big); + }, + ); + + it('should not dedupe different contents of the same size', async () => { + const src = createFixture('dedupe-same-size', { + 'a.txt': 'aaaa', + 'b.txt': 'bbbb', + }); + const dest = path.join(testRunDir, 'dedupe-same-size.asar'); + await createPackage(src, dest); + + expect(fileEntry(dest, 'b.txt').offset).not.toBe(fileEntry(dest, 'a.txt').offset); + expect(extractFile(dest, 'a.txt').toString()).toBe('aaaa'); + expect(extractFile(dest, 'b.txt').toString()).toBe('bbbb'); + }); + + it('should share contents between files that differ only by executable bit', async function () { + if (process.platform === 'win32') return; + const src = createFixture('dedupe-executable', { + 'script.sh': '#!/bin/bash\necho hi', + 'copy.sh': '#!/bin/bash\necho hi', + }); + fs.chmodSync(path.join(src, 'script.sh'), 0o755); + + const dest = path.join(testRunDir, 'dedupe-executable.asar'); + await createPackage(src, dest); + expect(fileEntry(dest, 'copy.sh').offset).toBe(fileEntry(dest, 'script.sh').offset); + + const extractDir = path.join(testRunDir, 'dedupe-executable-out'); + extractAll(dest, extractDir); + expect(fs.statSync(path.join(extractDir, 'script.sh')).mode & 0o111).toBeGreaterThan(0); + expect(fs.statSync(path.join(extractDir, 'copy.sh')).mode & 0o111).toBe(0); + }); + + it('should write every unpacked copy of duplicated contents', async () => { + const shared = 'shared native blob'; + const src = createFixture('dedupe-unpacked', { + 'a.node': shared, + 'dir/b.node': shared, + 'packed.txt': shared, + }); + const dest = path.join(testRunDir, 'dedupe-unpacked.asar'); + await createPackageWithOptions(src, dest, { unpack: '*.node' }); + + expect(fs.readFileSync(path.join(`${dest}.unpacked`, 'a.node'), 'utf8')).toBe(shared); + expect(fs.readFileSync(path.join(`${dest}.unpacked`, 'dir', 'b.node'), 'utf8')).toBe(shared); + // Unpacked files live outside the archive, so the packed copy is still stored + expect(extractFile(dest, 'packed.txt').toString()).toBe(shared); + }); + + it('should dedupe on the transformed contents', async () => { + const src = createFixture('dedupe-transform', { + 'a.txt': 'abc', + 'b.txt': 'cba', + 'c.txt': 'xyz', + }); + const dest = path.join(testRunDir, 'dedupe-transform.asar'); + // Sorting each file's bytes makes `a.txt` and `b.txt` identical once transformed + await createPackageWithOptions(src, dest, { + transform: () => + new Transform({ + transform(chunk, _encoding, callback) { + callback(null, Buffer.from([...chunk].sort((a, b) => a - b))); + }, + }), + }); + + expect(fileEntry(dest, 'b.txt').offset).toBe(fileEntry(dest, 'a.txt').offset); + expect(fileEntry(dest, 'c.txt').offset).not.toBe(fileEntry(dest, 'a.txt').offset); + expect(extractFile(dest, 'a.txt').toString()).toBe('abc'); + expect(extractFile(dest, 'b.txt').toString()).toBe('abc'); + expect(extractFile(dest, 'c.txt').toString()).toBe('xyz'); + }); + + it('should dedupe archives created from streams', async () => { + const shared = Buffer.from('shared stream contents'); + const streams = (): AsarStreamType[] => [ + { + type: 'file', + path: 'a.txt', + unpacked: false, + streamGenerator: () => Readable.from(shared), + stat: { mode: 0o644, size: shared.length }, + }, + { + type: 'file', + path: 'b.txt', + unpacked: false, + streamGenerator: () => Readable.from(shared), + stat: { mode: 0o644, size: shared.length }, + }, + ]; + + const dest = path.join(testRunDir, 'dedupe-streams.asar'); + await createPackageFromStreams(dest, streams()); + + expect(fileEntry(dest, 'b.txt').offset).toBe(fileEntry(dest, 'a.txt').offset); + expect(payloadSize(dest)).toBe(shared.length); + expect(extractFile(dest, 'b.txt')).toEqual(shared); + }); + + it('should keep empty files working alongside duplicates', async () => { + const src = createFixture('dedupe-empty', { + 'empty1.txt': '', + 'empty2.txt': '', + 'a.txt': 'dup', + 'b.txt': 'dup', + }); + const dest = path.join(testRunDir, 'dedupe-empty.asar'); + await createPackage(src, dest); + + expect(extractFile(dest, 'empty1.txt').length).toBe(0); + expect(extractFile(dest, 'empty2.txt').length).toBe(0); + expect(payloadSize(dest)).toBe('dup'.length); + expect(extractFile(dest, 'b.txt').toString()).toBe('dup'); + }); + }); });