feat: store duplicated file contents only once when packing

README.md

@@ -12,6 +12,7 @@ ASAR is a simple extensive archive format. It concatenates all files together wi
 * Support random access
 * Use JSON to store file information
 * Very easy to write a parser
+* Store the contents of duplicated files only once
 
 ## CLI
 
@@ -104,6 +105,18 @@ console.log('done.');
 
 Please note that there is currently **no** error handling provided!
 
+### Deduplication
+
+Files with identical contents are stored once and shared: the first copy is
+written into the archive and every other copy's header entry points at that same
+`offset`. Nothing changes for readers — each file still has its own entry, size,
+integrity hash, and executable bit — but archives with duplicated contents (a
+common shape for bundled `node_modules`) get smaller and pack faster, since the
+redundant bytes are never written.
+
+Unpacked files (`unpack` / `unpackDir`) are always written out in full, because
+they live on disk outside the archive.
+
 ### Transform
 
 You can pass in a `transform` option, that is a function, which either returns
@@ -200,6 +213,9 @@ Structure of `header` is something like this:
 `offset` starts from 0 so you have to manually add the size of `header_size` and
 `header` to the `offset` to get the real offset of the file.
 
+Files with identical contents share a single copy in the archive, so more than
+one entry can point at the same `offset`.
+
 `offset` is a UINT64 number represented in string, because there is no way to
 precisely represent UINT64 in JavaScript `Number`. `size` is a JavaScript
 `Number` that is no larger than `Number.MAX_SAFE_INTEGER`, which has a value of

benchmark/generate-fixtures.ts

@@ -4,6 +4,9 @@ import crypto from 'node:crypto';
 
 const BENCHMARK_DIR = path.join(import.meta.dirname, 'fixtures');
 
+/** How many distinct contents duplicated files are drawn from */
+const DUPLICATE_POOL_SIZE = 64;
+
 export type FixtureConfig = {
   name: string;
   fileCount: number;
@@ -13,6 +16,8 @@ export type FixtureConfig = {
   depth: number;
   /** Number of subdirectories per level */
   breadth: number;
+  /** Fraction of files that reuse the contents of an earlier file (0 by default) */
+  duplicateRatio?: number;
 };
 
 export const FIXTURES: FixtureConfig[] = [
@@ -22,6 +27,14 @@ export const FIXTURES: FixtureConfig[] = [
   { name: 'few-large-files', fileCount: 20, avgFileSize: 1024 * 1024, depth: 2, breadth: 2 },
   { name: 'many-small-files', fileCount: 10000, avgFileSize: 256, depth: 3, breadth: 10 },
   { name: 'deep-tree', fileCount: 1000, avgFileSize: 2048, depth: 10, breadth: 2 },
+  {
+    name: 'duplicate-heavy',
+    fileCount: 3000,
+    avgFileSize: 4096,
+    depth: 4,
+    breadth: 4,
+    duplicateRatio: 0.5,
+  },
 ];
 
 function generateRandomContent(size: number): Buffer {
@@ -59,13 +72,25 @@ export function generateFixture(config: FixtureConfig): string {
     }
   }
 
+  // Contents that duplicated files are drawn from, mimicking the repeated
+  // licenses/READMEs/vendored copies that show up across `node_modules`
+  const duplicateRatio = config.duplicateRatio ?? 0;
+  const duplicatePool: Buffer[] = [];
+
   // Distribute files across directories
   for (let i = 0; i < config.fileCount; i++) {
     const dir = dirs[i % dirs.length];
-    // Vary file sizes: 50% to 150% of average
-    const sizeVariation = 0.5 + Math.random();
-    const size = Math.max(1, Math.floor(config.avgFileSize * sizeVariation));
-    const content = generateRandomContent(size);
+    let content: Buffer;
+    if (duplicatePool.length > 0 && Math.random() < duplicateRatio) {
+      content = duplicatePool[i % duplicatePool.length];
+    } else {
+      // Vary file sizes: 50% to 150% of average
+      const sizeVariation = 0.5 + Math.random();
+      content = generateRandomContent(Math.max(1, Math.floor(config.avgFileSize * sizeVariation)));
+      if (duplicatePool.length < DUPLICATE_POOL_SIZE) {
+        duplicatePool.push(content);
+      }
+    }
     const ext = ['.txt', '.js', '.json', '.bin', '.dat'][i % 5];
     const filePath = path.join(fixtureDir, dir, `file_${i}${ext}`);
     fs.writeFileSync(filePath, content);

src/asar.ts

@@ -195,22 +195,23 @@ export async function createPackageFromFiles(
         );
         filesystem.insertDirectory(filename, shouldUnpack);
         break;
-      case 'file':
+      case 'file': {
         shouldUnpack = shouldUnpackPath(
           filename,
           path.relative(src, path.dirname(filename)),
           options.unpack,
           options.unpackDir,
         );
-        files.push({ filename, unpack: shouldUnpack });
-        await filesystem.insertFile(
+        const duplicate = await filesystem.insertFile(
           filename,
           () => fs.createReadStream(filename),
           shouldUnpack,
           file,
           options,
         );
+        files.push({ filename, unpack: shouldUnpack, duplicate });
         break;
+      }
       case 'link':
         shouldUnpack = shouldUnpackPath(
           filename,
@@ -277,18 +278,23 @@ export async function createPackageFromStreams(dest: string, streams: AsarStream
       case 'directory':
         filesystem.insertDirectory(filename, stream.unpacked);
         break;
-      case 'file':
+      case 'file': {
+        const duplicate = await filesystem.insertFile(
+          filename,
+          stream.streamGenerator,
+          stream.unpacked,
+          { type: 'file', stat: stream.stat },
+        );
         files.push({
           filename,
           streamGenerator: stream.streamGenerator,
           link: undefined,
           mode: stream.stat.mode,
           unpack: stream.unpacked,
+          duplicate,
         });
-        return filesystem.insertFile(filename, stream.streamGenerator, stream.unpacked, {
-          type: 'file',
-          stat: stream.stat,
-        });
+        break;
+      }
       case 'link':
         links.push({
           filename,

src/disk.ts

@@ -183,6 +183,8 @@
 export type BasicFilesArray = {
   filename: string;
   unpack: boolean;
+  /** Contents shared with an earlier file, already written to the archive */
+  duplicate?: boolean;
 }[];
 
 export type BasicStreamArray = {
@@ -191,6 +193,8 @@
   mode: Stats['mode'];
   unpack: boolean;
   link: string | undefined; // only for symlinks, should refactor as part of larger project refactor in follow-up PR
+  /** Contents shared with an earlier file, already written to the archive */
+  duplicate?: boolean;
 }[];
 
 export type FilesystemFilesAndLinks<T extends BasicFilesArray | BasicStreamArray> = {
@@ -221,6 +225,9 @@
   };
 
   for (const file of files) {
+    if (file.duplicate) {
+      continue;
+    }
     if (file.unpack) {
       await flushPendingBuffers();
       const filename = path.relative(filesystem.getRootPath(), file.filename);
@@ -269,7 +276,10 @@
   const out = await createFilesystemWriteStream(filesystem, dest);
 
   const { files, links } = lists;
   for await (const file of files) {
+    if (file.duplicate) {
+      continue;
+    }
     // the file should not be packed into archive
     if (file.unpack) {
       const targetFile = path.join(`${dest}.unpacked`, file.filename);

src/filesystem.ts

@@ -40,12 +40,15 @@ export class Filesystem {
   private header: FilesystemEntry;
   private headerSize: number;
   private offset: bigint;
+  // SHA-256 of contents already stored => the offset those contents live at
+  private contentOffsets: Map<string, string>;
 
   constructor(src: string) {
     this.src = path.resolve(src);
     this.header = { files: Object.create(null) };
     this.headerSize = 0;
     this.offset = BigInt(0);
+    this.contentOffsets = new Map();
   }
 
   getRootPath() {
@@ -116,14 +119,17 @@ export class Filesystem {
     options: {
       transform?: (filePath: string) => NodeJS.ReadWriteStream | void;
     } = {},
-  ): Promise<void> {
+  ): Promise<boolean> {
     const dirNode = this.searchNodeFromPath(path.dirname(p)) as FilesystemDirectoryEntry;
     const node = this.searchNodeFromPath(p) as FilesystemFileEntry;
     if (shouldUnpack || dirNode.unpacked) {
       node.size = file.stat.size;
       node.unpacked = true;
+      // Unpacked files are copied out of the archive as standalone files, so every
+      // copy is written even when their contents are identical.
       return getFileIntegrity(streamGenerator()).then((integrity) => {
         node.integrity = integrity;
+        return false;
       });
     }
 
@@ -139,29 +145,58 @@ export class Filesystem {
       throw new Error(`${p}: file size can not be larger than 4.2GB`);
     }
 
-    node.size = size;
-    node.offset = this.offset.toString();
-    if (process.platform !== 'win32' && file.stat.mode & 0o100) {
-      node.executable = true;
-    }
+    const executable = process.platform !== 'win32' && (file.stat.mode & 0o100) !== 0;
 
     if (size <= BUFFER_HASH_THRESHOLD) {
       // Fully synchronous fast path — no Promise, no stream, no microtask yield
       try {
         const fileBuffer = fs.readFileSync(p);
-        node.integrity = getFileIntegrityFromBuffer(fileBuffer);
-        file.cachedBuffer = fileBuffer;
-        this.offset += BigInt(size);
-        return Promise.resolve();
+        const integrity = getFileIntegrityFromBuffer(fileBuffer);
+        const duplicate = this.storeFileEntry(node, size, executable, integrity);
+        if (!duplicate) {
+          file.cachedBuffer = fileBuffer;
+        }
+        return Promise.resolve(duplicate);
       } catch {
         // Fall through to stream path
       }
     }
 
-    return getFileIntegrity(streamGenerator()).then((integrity) => {
-      node.integrity = integrity;
-      this.offset += BigInt(size);
-    });
+    return getFileIntegrity(streamGenerator()).then((integrity) =>
+      this.storeFileEntry(node, size, executable, integrity),
+    );
+  }
+
+  /**
+   * Fills in a file entry and reserves the region of the archive that holds its
+   * contents, growing the archive by `size` bytes. When a file with identical
+   * contents was already inserted, the entry instead points at the offset of those
+   * contents and the archive is left unchanged — the caller must then skip writing
+   * the contents.
+   *
+   * @returns whether the contents were already stored by an earlier file
+   */
+  private storeFileEntry(
+    node: FilesystemFileEntry,
+    size: number,
+    executable: boolean,
+    integrity: FileIntegrity,
+  ): boolean {
+    const sharedOffset = this.contentOffsets.get(integrity.hash);
+
+    node.size = size;
+    node.offset = sharedOffset ?? this.offset.toString();
+    if (executable) {
+      node.executable = true;
+    }
+    node.integrity = integrity;
+
+    if (sharedOffset !== undefined) {
+      return true;
+    }
+    this.contentOffsets.set(integrity.hash, node.offset);
+    this.offset += BigInt(size);
+    return false;
   }
 
   private async insertFileAsync(
@@ -170,7 +205,7 @@ export class Filesystem {
     file: CrawledFileType,
     node: FilesystemFileEntry,
     transformed: NodeJS.ReadWriteStream,
-  ) {
+  ): Promise<boolean> {
     const tmpdir = await fs.mkdtemp(path.join(os.tmpdir(), 'asar-'));
     const tmpfile = path.join(tmpdir, path.basename(p));
     const out = fs.createWriteStream(tmpfile);
@@ -186,15 +221,17 @@ export class Filesystem {
       throw new Error(`${p}: file size can not be larger than 4.2GB`);
     }
 
-    node.size = size;
-    node.offset = this.offset.toString();
-    if (process.platform !== 'win32' && file.stat.mode & 0o100) {
-      node.executable = true;
-    }
     // Integrity must be computed over the transformed bytes that are actually
     // stored in the archive, not the original (pre-transform) source bytes.
-    node.integrity = await getFileIntegrity(fs.createReadStream(file.transformed.path));
-    this.offset += BigInt(size);
+    const integrity = await getFileIntegrity(fs.createReadStream(file.transformed.path));
+    const executable = process.platform !== 'win32' && (file.stat.mode & 0o100) !== 0;
+    const duplicate = this.storeFileEntry(node, size, executable, integrity);
+    if (duplicate) {
+      // Nothing will read these transformed bytes again, so don't leave them behind
+      file.transformed = undefined;
+      fs.rmSync(tmpdir, { recursive: true, force: true });
+    }
+    return duplicate;
   }
 
   insertLink(