diff --git a/CHANGELOG.md b/CHANGELOG.md index 5383285..b2ef141 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +* `VectorDatabase::compact()` rebuilds the HNSW graph from the active documents using their stored vectors and returns the number of reclaimed nodes. Soft-deleted nodes, including the ones left behind by `updateDocument()`, no longer accumulate in memory and in `hnsw.bin` for the lifetime of the database. Active documents keep their internal node IDs, so no document file is renamed. On a persistent database the result is saved under the folder lock. +* `VectorDatabase::deletedCount()` reports how many soft-deleted nodes the graph holds, so callers can compact above a threshold. +* `HNSW\Index::compact()` and `HNSW\Index::deletedCount()`. +* `HNSW\Index::insert()` accepts an optional explicit node ID, which must be greater than every ID in the graph. + +### Fixed +* The README claimed that document files are removed from disk as soon as a document is deleted. Removal is deferred to the next `save()`. + ## [0.4.1] - 2026-09-02 Documentation only. No code changes, no behaviour changes. diff --git a/README.md b/README.md index 780315f..2e1953e 100644 --- a/README.md +++ b/README.md @@ -324,6 +324,7 @@ $results = $db->vectorSearch($queryVector, k: 10); ```php $db->count(); // Active (non-deleted) documents currently indexed. +$db->deletedCount(); // Soft-deleted documents still held in the HNSW graph. $db->isPersistent(); // false for in-memory databases, where save() would throw. ``` @@ -401,7 +402,20 @@ $updated = $db->updateDocument(new Document( $db->save(); ``` -Deleted documents are soft-deleted from the HNSW graph (kept for connectivity but excluded from results) and fully removed from the BM25 index. Document files are deleted from disk immediately. +Deleted documents are soft-deleted from the HNSW graph (kept for connectivity but excluded from results) and fully removed from the BM25 index. On a persistent database a tombstone is written immediately and the document file is removed on the next `save()`. + +### Reclaiming deleted documents + +A soft-deleted node keeps its vector and its edges, in memory and in `hnsw.bin`. Since `updateDocument()` is a delete plus an insert, the graph grows with the total number of edits rather than with the number of live documents. `compact()` rebuilds the HNSW graph from the active documents using their stored vectors, so nothing needs to be embedded again: + +```php +// Compact once deleted nodes exceed 20% of the live documents. +if ($db->deletedCount() > $db->count() * 0.2) { + $reclaimed = $db->compact(); // number of nodes removed from the graph +} +``` + +On a persistent database `compact()` saves the result itself, under the same lock as `save()`. When nothing is deleted it returns 0 and writes nothing. The rebuild costs about as much as inserting every active document again, so run it from a maintenance job rather than on every request. ## Metadata filtering diff --git a/src/HNSW/Index.php b/src/HNSW/Index.php index 40ceb49..7d6d03e 100644 --- a/src/HNSW/Index.php +++ b/src/HNSW/Index.php @@ -154,10 +154,28 @@ public function __construct(private readonly Config $config = new Config()) /** * Insert a document into the index. * + * When `$nodeId` is null the next ID after the highest one currently in the + * graph is assigned. VectorDatabase passes its own ID instead, so the two + * stay aligned after compact() leaves gaps in the ID space. + * + * @param int|null $nodeId Explicit node ID, greater than every ID in the graph. + * * @throws DimensionMismatchException if the vector dimension differs from previously inserted vectors. + * @throws \InvalidArgumentException if `$nodeId` is not greater than every ID in the graph. */ - public function insert(Document $document): void + public function insert(Document $document, ?int $nodeId = null): void { + $lastId = array_key_last($this->nodes); + if ($nodeId === null) { + $nodeId = $lastId === null ? 0 : $lastId + 1; + } elseif ($nodeId < 0 || ($lastId !== null && $nodeId <= $lastId)) { + throw new \InvalidArgumentException(sprintf( + 'Node ID %d must be greater than the highest node ID in the graph (%s).', + $nodeId, + $lastId === null ? 'none' : (string) $lastId, + )); + } + $vector = $document->vector; $dim = count($vector); @@ -167,8 +185,9 @@ public function insert(Document $document): void throw DimensionMismatchException::forVectors($this->dimension, $dim); } - $nodeId = count($this->nodes); - $maxLayer = $this->randomLevel(); + // Graph size before this insert; drives the adaptive efConstruction below. + $graphSize = count($this->nodes); + $maxLayer = $this->randomLevel(); $node = new Node($nodeId, $vector, $maxLayer); $this->nodes[$nodeId] = $node; @@ -213,7 +232,7 @@ public function insert(Document $document): void // base layer and M on upper layers, even during early inserts. $efC = max( $mMax, - min($this->config->efConstruction, (int) ($nodeId / 10)), + min($this->config->efConstruction, (int) ($graphSize / 10)), ); // Find efConstruction nearest neighbours at this layer. $W = $this->searchLayer($dv, [[$epDist, $ep]], $efC, $lc); @@ -432,6 +451,59 @@ public function delete(int $nodeId): bool return true; } + /** + * Number of soft-deleted nodes still held in the graph. + */ + public function deletedCount(): int + { + return count($this->deleted); + } + + /** + * Physically remove every soft-deleted node by rebuilding the graph from + * the active nodes and their stored vectors. + * + * Active nodes keep their node IDs, so callers holding node IDs (such as + * VectorDatabase) need no remapping. The rebuild re-inserts every active + * node, costing about as much as building the index from scratch, but it + * needs no re-embedding. + * + * @return int Number of nodes reclaimed. + */ + public function compact(): int + { + $reclaimed = count($this->deleted); + if ($reclaimed === 0) { + return 0; + } + + $active = []; + foreach ($this->nodes as $nodeId => $node) { + if (!isset($this->deleted[$nodeId])) { + $doc = $this->documents[$nodeId]; + $active[$nodeId] = new Document( + id: $doc->id, + vector: $node->vector, + text: $doc->text, + metadata: $doc->metadata, + ); + } + } + + $this->nodes = []; + $this->documents = []; + $this->distVectors = []; + $this->deleted = []; + $this->entryPoint = null; + $this->maxLayer = 0; + + foreach ($active as $nodeId => $doc) { + $this->insert($doc, $nodeId); + } + + return $reclaimed; + } + /** * Check if a node has been soft-deleted. */ diff --git a/src/VectorDatabase.php b/src/VectorDatabase.php index 10748ed..0794996 100644 --- a/src/VectorDatabase.php +++ b/src/VectorDatabase.php @@ -183,7 +183,7 @@ public function addDocument(Document $document): void $this->nodeIdToDoc[$nodeId] = $document; $this->docIdToNodeId[$document->id] = $nodeId; - $this->hnswIndex->insert($document); + $this->hnswIndex->insert($document, $nodeId); $this->bm25Index->addDocument($nodeId, $document); // Persist doc file asynchronously when a path is configured. @@ -215,7 +215,9 @@ public function addDocuments(array $documents): void * Delete a document by its user-visible ID. * * The document is soft-deleted from HNSW (excluded from results but kept - * for graph connectivity) and fully removed from the BM25 index. + * for graph connectivity) and fully removed from the BM25 index. The HNSW + * node, vector included, stays in memory and in `hnsw.bin` until + * `compact()` is called. * * When persistence is enabled, a tombstone marker is written immediately so * the deletion survives a crash. The physical doc file is removed during @@ -863,6 +865,46 @@ public function count(): int return $this->hnswIndex->count(); } + /** + * Number of soft-deleted documents still held in the HNSW graph. + * + * Every `deleteDocument()` and `updateDocument()` adds one. Use it to + * decide when to call `compact()`, for example once it exceeds a share + * of `count()`. + */ + public function deletedCount(): int + { + return $this->hnswIndex->deletedCount(); + } + + /** + * Reclaim the space held by soft-deleted documents. + * + * Rebuilds the HNSW graph from the active documents using their stored + * vectors, so nothing is re-embedded. The BM25 index needs no rebuild + * because deletions already remove documents from it. Active documents + * keep their internal node IDs, so no document file is renamed. + * + * The rebuild costs about as much as inserting every active document + * again. When persistence is enabled and something was reclaimed, the + * database is saved afterwards, with the same locking as `save()`. + * + * @return int Number of reclaimed documents (0 when there was nothing to do). + * + * @throws LockTimeoutException if the folder lock is held by another process. + * @throws \RuntimeException on I/O failure while saving. + */ + public function compact(): int + { + $reclaimed = $this->hnswIndex->compact(); + + if ($reclaimed > 0 && $this->path !== null) { + $this->save(); + } + + return $reclaimed; + } + // ------------------------------------------------------------------ // Fusion strategies // ------------------------------------------------------------------ diff --git a/tests/HNSW/IndexDeleteTest.php b/tests/HNSW/IndexDeleteTest.php index 949aaf6..a051e07 100644 --- a/tests/HNSW/IndexDeleteTest.php +++ b/tests/HNSW/IndexDeleteTest.php @@ -159,4 +159,121 @@ public function testChurnPreservesRecall(): void sprintf('Churn recall too low: %d/%d', $recalled, $k), ); } + + public function testDeletedCount(): void + { + $index = $this->makeIndex(); + $index->insert(new Document(id: 'a', vector: [1.0, 0.0])); + $index->insert(new Document(id: 'b', vector: [0.0, 1.0])); + self::assertSame(0, $index->deletedCount()); + + $index->delete(0); + self::assertSame(1, $index->deletedCount()); + + $index->delete(0); + self::assertSame(1, $index->deletedCount(), 'Deleting twice counts once.'); + } + + public function testCompactRemovesDeletedNodesAndKeepsIds(): void + { + $index = $this->makeIndex(); + $index->insert(new Document(id: 'a', vector: [1.0, 0.0])); // node 0 + $index->insert(new Document(id: 'b', vector: [0.0, 1.0])); // node 1 + $index->insert(new Document(id: 'c', vector: [-1.0, 0.0])); // node 2 + $index->delete(1); + + self::assertSame(1, $index->compact()); + self::assertSame(0, $index->deletedCount()); + self::assertSame(2, $index->count()); + + // Survivors keep their node IDs; the reclaimed node is gone for good. + self::assertSame([1.0, 0.0], $index->getVector(0)); + self::assertSame([-1.0, 0.0], $index->getVector(2)); + self::assertFalse($index->delete(1)); + + $state = $index->exportState(); + self::assertSame([0, 2], array_keys($state['nodes'])); + self::assertSame([], $state['deleted']); + foreach ($state['nodes'] as $node) { + foreach ($node['connections'] as $conns) { + self::assertNotContains(1, $conns, 'No edge may point at a reclaimed node.'); + } + } + } + + public function testCompactWithoutDeletionsIsNoop(): void + { + $index = $this->makeIndex(); + $index->insert(new Document(id: 'a', vector: [1.0, 0.0])); + $before = $index->exportState(); + + self::assertSame(0, $index->compact()); + self::assertSame($before, $index->exportState()); + } + + public function testInsertAfterCompactUsesNextFreeId(): void + { + $index = $this->makeIndex(); + $index->insert(new Document(id: 'a', vector: [1.0, 0.0])); // node 0 + $index->insert(new Document(id: 'b', vector: [0.0, 1.0])); // node 1 + $index->delete(0); + $index->compact(); + + $index->insert(new Document(id: 'c', vector: [-1.0, 0.0])); + self::assertSame([1, 2], array_keys($index->exportState()['nodes'])); + } + + public function testInsertAcceptsExplicitIdWithGap(): void + { + $index = $this->makeIndex(); + $index->insert(new Document(id: 'a', vector: [1.0, 0.0]), 5); + $index->insert(new Document(id: 'b', vector: [0.0, 1.0]), 9); + + self::assertSame([5, 9], array_keys($index->exportState()['nodes'])); + self::assertSame('b', $index->search([0.0, 1.0], 1)[0]->document->id); + } + + public function testInsertRejectsExplicitIdNotAboveHighest(): void + { + $index = $this->makeIndex(); + $index->insert(new Document(id: 'a', vector: [1.0, 0.0]), 3); + + $this->expectException(\InvalidArgumentException::class); + $index->insert(new Document(id: 'b', vector: [0.0, 1.0]), 3); + } + + public function testCompactPreservesRecall(): void + { + mt_srand(42); + $index = $this->makeIndex(); + $dim = 8; + $k = 10; + + $vectors = []; + for ($i = 0; $i < 300; $i++) { + $vectors[$i] = $this->randomVector($dim); + $index->insert(new Document(id: $i, vector: $vectors[$i])); + } + for ($i = 0; $i < 300; $i += 3) { + $index->delete($i); + unset($vectors[$i]); + } + + self::assertSame(100, $index->compact()); + + $query = $this->randomVector($dim); + $hnswIds = array_map(fn($sr) => $sr->document->id, $index->search($query, $k)); + + $dists = []; + foreach ($vectors as $id => $v) { + $dists[$id] = sqrt(array_sum(array_map(fn($a, $b) => ($a - $b) ** 2, $query, $v))); + } + asort($dists); + $groundTruth = array_slice(array_keys($dists), 0, $k); + + self::assertGreaterThanOrEqual( + (int) ceil($k * 0.7), + count(array_intersect($hnswIds, $groundTruth)), + ); + } } diff --git a/tests/PersistenceTest.php b/tests/PersistenceTest.php index 53deeff..aabc4f3 100644 --- a/tests/PersistenceTest.php +++ b/tests/PersistenceTest.php @@ -467,4 +467,86 @@ public function testUpdateDocumentPersistsCorrectly(): void self::assertSame('updated content', $results[0]->document->text); self::assertSame(['version' => 2], $results[0]->document->metadata); } + + // ------------------------------------------------------------------ + // Compaction + // ------------------------------------------------------------------ + + public function testCompactShrinksHnswFileAndSaves(): void + { + $db = $this->makeDb(); + for ($i = 0; $i < 30; $i++) { + $db->addDocument(new Document(id: $i, vector: [cos($i), sin($i)], text: "doc {$i}")); + } + $db->save(); + + // Replace every document once, as an application re-indexing its content would. + for ($i = 0; $i < 30; $i++) { + $db->updateDocument(new Document(id: $i, vector: [sin($i), cos($i)], text: "doc {$i} v2")); + } + $db->save(); + clearstatcache(); + $bloated = filesize($this->tmpDir . '/hnsw.bin'); + + self::assertSame(30, $db->compact()); + clearstatcache(); + self::assertLessThan($bloated, filesize($this->tmpDir . '/hnsw.bin')); + + // compact() saved on its own: a fresh open() sees the compacted state. + $meta = json_decode((string) file_get_contents($this->tmpDir . '/meta.json'), true); + self::assertSame([], $meta['deleted']); + + $loaded = $this->openDb(); + self::assertSame(30, $loaded->count()); + self::assertSame(0, $loaded->deletedCount()); + $top = $loaded->vectorSearch([sin(7), cos(7)], k: 1); + self::assertSame(7, $top[0]->document->id); + self::assertSame('doc 7 v2', $top[0]->document->text); + } + + public function testCompactOnReopenedDatabaseKeepsDocFiles(): void + { + $db = $this->makeDb(); + $db->addDocument(new Document(id: 'a', vector: [1.0, 0.0], text: 'alpha', metadata: ['n' => 1])); + $db->addDocument(new Document(id: 'b', vector: [0.0, 1.0], text: 'beta', metadata: ['n' => 2])); + $db->addDocument(new Document(id: 'c', vector: [-1.0, 0.0], text: 'gamma', metadata: ['n' => 3])); + $db->deleteDocument('b'); + $db->save(); + + $loaded = $this->openDb(); + self::assertSame(1, $loaded->deletedCount()); + self::assertSame(1, $loaded->compact()); + + // Survivors keep their node IDs, so their doc files are untouched. + self::assertFileExists($this->tmpDir . '/docs/0.bin'); + self::assertFileDoesNotExist($this->tmpDir . '/docs/1.bin'); + self::assertFileExists($this->tmpDir . '/docs/2.bin'); + + // New documents continue after the highest ID ever assigned. + $loaded->addDocument(new Document(id: 'd', vector: [0.0, -1.0], text: 'delta')); + $loaded->save(); + + $reloaded = $this->openDb(); + self::assertSame(3, $reloaded->count()); + $top = $reloaded->vectorSearch([-1.0, 0.0], k: 1); + self::assertSame('c', $top[0]->document->id); + self::assertSame(['n' => 3], $top[0]->document->metadata); + self::assertSame('d', $reloaded->textSearch('delta', k: 1)[0]->document->id); + self::assertFileExists($this->tmpDir . '/docs/3.bin'); + } + + public function testCompactWithoutDeletionsDoesNotSave(): void + { + $db = $this->makeDb(); + $db->addDocument(new Document(id: 1, vector: [1.0, 0.0], text: 'one')); + $db->save(); + clearstatcache(); + $inode = fileinode($this->tmpDir . '/hnsw.bin'); + + self::assertSame(0, $db->compact()); + + // save() replaces hnsw.bin through a rename, which would change the inode. + clearstatcache(); + self::assertSame($inode, fileinode($this->tmpDir . '/hnsw.bin')); + } } diff --git a/tests/VectorDatabaseTest.php b/tests/VectorDatabaseTest.php index d3d8605..785b88d 100644 --- a/tests/VectorDatabaseTest.php +++ b/tests/VectorDatabaseTest.php @@ -377,6 +377,113 @@ public function testUpdateDocumentWithoutIdThrows(): void $db->updateDocument(new Document(vector: [0.0, 1.0], text: 'no id')); } + // ------------------------------------------------------------------ + // Compaction + // ------------------------------------------------------------------ + + public function testDeletedCountTracksDeletesAndUpdates(): void + { + $db = $this->makeDb(); + $db->addDocument(new Document(id: 1, vector: [1.0, 0.0], text: 'one')); + $db->addDocument(new Document(id: 2, vector: [0.0, 1.0], text: 'two')); + self::assertSame(0, $db->deletedCount()); + + $db->deleteDocument(1); + self::assertSame(1, $db->deletedCount()); + + $db->updateDocument(new Document(id: 2, vector: [0.5, 0.5], text: 'two again')); + self::assertSame(2, $db->deletedCount()); + self::assertSame(1, $db->count()); + } + + public function testCompactWithoutDeletionsReturnsZero(): void + { + $db = $this->makeDb(); + self::assertSame(0, $db->compact()); + + $db->addDocument(new Document(id: 1, vector: [1.0, 0.0], text: 'one')); + self::assertSame(0, $db->compact()); + self::assertSame(1, $db->count()); + } + + public function testCompactReclaimsDeletedDocuments(): void + { + $db = $this->makeDb(); + for ($i = 0; $i < 20; $i++) { + $db->addDocument(new Document(id: $i, vector: [cos($i), sin($i)], text: "doc number {$i}")); + } + for ($i = 0; $i < 20; $i += 2) { + $db->deleteDocument($i); + } + + self::assertSame(10, $db->compact()); + self::assertSame(0, $db->deletedCount()); + self::assertSame(10, $db->count()); + self::assertSame(0, $db->compact(), 'A second compaction has nothing left to reclaim.'); + } + + public function testSearchWorksAfterCompact(): void + { + $db = $this->makeDb(); + for ($i = 0; $i < 20; $i++) { + $db->addDocument(new Document(id: $i, vector: [cos($i), sin($i)], text: "doc number {$i}")); + } + for ($i = 0; $i < 20; $i += 2) { + $db->deleteDocument($i); + } + $db->compact(); + + $vectorIds = array_map(fn($r) => $r->document->id, $db->vectorSearch([cos(3), sin(3)], k: 20)); + sort($vectorIds); + self::assertSame([1, 3, 5, 7, 9, 11, 13, 15, 17, 19], $vectorIds); + + $top = $db->vectorSearch([cos(3), sin(3)], k: 1); + self::assertSame(3, $top[0]->document->id); + self::assertSame('doc number 3', $top[0]->document->text); + + $textIds = array_map(fn($r) => $r->document->id, $db->textSearch('number', k: 20)); + sort($textIds); + self::assertSame([1, 3, 5, 7, 9, 11, 13, 15, 17, 19], $textIds); + + $hybrid = $db->hybridSearch(vector: [cos(5), sin(5)], text: 'number', k: 1); + self::assertSame(5, $hybrid[0]->document->id); + } + + public function testAddAndDeleteAfterCompact(): void + { + $db = $this->makeDb(); + $db->addDocument(new Document(id: 'a', vector: [1.0, 0.0], text: 'alpha')); + $db->addDocument(new Document(id: 'b', vector: [0.0, 1.0], text: 'beta')); + $db->addDocument(new Document(id: 'c', vector: [-1.0, 0.0], text: 'gamma')); + $db->deleteDocument('c'); + $db->compact(); + + $db->addDocument(new Document(id: 'd', vector: [0.0, -1.0], text: 'delta')); + self::assertSame('d', $db->vectorSearch([0.0, -1.0], k: 1)[0]->document->id); + + self::assertTrue($db->deleteDocument('a')); + self::assertSame(['b', 'd'], array_map( + fn($r) => $r->document->id, + $db->metadataSearch(), + )); + } + + public function testCompactAfterDeletingEverything(): void + { + $db = $this->makeDb(); + $db->addDocument(new Document(id: 1, vector: [1.0, 0.0], text: 'one')); + $db->addDocument(new Document(id: 2, vector: [0.0, 1.0], text: 'two')); + $db->deleteDocument(1); + $db->deleteDocument(2); + + self::assertSame(2, $db->compact()); + self::assertSame(0, $db->count()); + self::assertSame([], $db->vectorSearch([1.0, 0.0], k: 5)); + + $db->addDocument(new Document(id: 3, vector: [1.0, 0.0], text: 'three')); + self::assertSame(3, $db->vectorSearch([1.0, 0.0], k: 1)[0]->document->id); + } + // ------------------------------------------------------------------ // Vector search with metadata filters // ------------------------------------------------------------------