From 38d798cfaac16c4198a540d1c23d249b37027f4b Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Thu, 3 Sep 2026 21:11:25 +0300 Subject: [PATCH] fix: repair orphaned nodes in hnsw graph after construction --- knn/CMakeLists.txt | 12 +- knn/graphrepair.cpp | 386 ++++++++++++++++++++++++++++ knn/graphrepair.h | 67 +++++ knn/knn.cpp | 5 + knn/test_graphrepair.cpp | 530 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 999 insertions(+), 1 deletion(-) create mode 100644 knn/graphrepair.cpp create mode 100644 knn/graphrepair.h create mode 100644 knn/test_graphrepair.cpp diff --git a/knn/CMakeLists.txt b/knn/CMakeLists.txt index d7ae9a6c..af855cae 100644 --- a/knn/CMakeLists.txt +++ b/knn/CMakeLists.txt @@ -18,12 +18,22 @@ cmake_minimum_required ( VERSION 3.21 ) # because of IMPORTED_RUNTIME_ARTIFACTS include ( GetHNSW ) -add_library ( knn_lib MODULE knn.cpp knn.h iterator.cpp iterator.h embeddings.cpp embeddings.h quantizer.cpp quantizer.h quantile.cpp quantile.h space.cpp space.h termination.cpp termination.h ${columnar_SOURCE_DIR}/embeddings/manticoresearch_text_embeddings.h ) +set ( KNN_SOURCES knn.cpp knn.h iterator.cpp iterator.h embeddings.cpp embeddings.h quantizer.cpp quantizer.h quantile.cpp quantile.h space.cpp space.h termination.cpp termination.h graphrepair.cpp graphrepair.h ) + +add_library ( knn_lib MODULE ${KNN_SOURCES} ${columnar_SOURCE_DIR}/embeddings/manticoresearch_text_embeddings.h ) target_include_directories(knn_lib PRIVATE ${columnar_SOURCE_DIR}/embeddings) target_link_libraries ( knn_lib PRIVATE hnswlib::hnswlib columnar_root util common ) set_target_properties( knn_lib PROPERTIES PREFIX "" OUTPUT_NAME lib_manticore_knn${lib_arch_suffix} ) +# Regression test for the HNSW connectivity repair pass. Links the knn sources directly it can reach graphrepair.h (internal, not part of the exported C ABI) +if ( NOT EXTERNAL_LIB AND BUILD_TESTING ) + add_executable ( knn_graphrepair_test test_graphrepair.cpp ${KNN_SOURCES} ) + target_include_directories ( knn_graphrepair_test PRIVATE ${columnar_SOURCE_DIR}/embeddings ${CMAKE_CURRENT_SOURCE_DIR} ) + target_link_libraries ( knn_graphrepair_test PRIVATE hnswlib::hnswlib columnar_root util common ) + add_test ( NAME knn_graphrepair COMMAND knn_graphrepair_test ) +endif() + # Try to find manticoresearch text embeddings library message(STATUS "Looking for manticoresearch text embeddings library...") diff --git a/knn/graphrepair.cpp b/knn/graphrepair.cpp new file mode 100644 index 00000000..7d20d10d --- /dev/null +++ b/knn/graphrepair.cpp @@ -0,0 +1,386 @@ +// Copyright (c) 2026, Manticore Software LTD (https://manticoresearch.com) +// All rights reserved +// +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "graphrepair.h" + +#include +#include +#include +#include + +namespace knn +{ + +namespace +{ + +using HNSWAlg_t = hnswlib::HierarchicalNSW; + +static const hnswlib::tableint INVALID_NODE = std::numeric_limits::max(); + +// level-0 neighbour view of one node +struct LinkList_t +{ + hnswlib::linklistsizeint * m_pHeader = nullptr; + hnswlib::tableint * m_pLinks = nullptr; + uint32_t m_uCount = 0; +}; + + +inline LinkList_t GetLinks0 ( const HNSWAlg_t & tAlg, hnswlib::tableint tId ) +{ + LinkList_t tRes; + tRes.m_pHeader = tAlg.get_linklist0(tId); + tRes.m_pLinks = (hnswlib::tableint*)( tRes.m_pHeader+1 ); + tRes.m_uCount = tAlg.getListCount ( tRes.m_pHeader ); + return tRes; +} + + +// Distance with tFrom as the query and tTo as the document +inline float RepairDist ( const HNSWAlg_t & tAlg, hnswlib::tableint tFrom, hnswlib::tableint tTo, uint64_t & uDistCalls ) +{ + uDistCalls++; + return tAlg.calcDistanceConstruction<> ( tAlg.getDataByInternalId(tFrom), tTo, tAlg.getExternalLabel(tFrom) ); +} + + +// Appends into a free slot +inline void AppendLink ( const HNSWAlg_t & tAlg, hnswlib::tableint tTarget, hnswlib::tableint tNew ) +{ + LinkList_t tLL = GetLinks0 ( tAlg, tTarget ); + assert ( tLL.m_uCount < tAlg.maxM0_ ); + tLL.m_pLinks[tLL.m_uCount] = tNew; + tAlg.setListCount ( tLL.m_pHeader, (unsigned short)( tLL.m_uCount+1 ) ); +} + + +class NodeBitmap_c +{ +public: + void Resize ( uint64_t uSize ) { m_dBits.assign ( (size_t)( ( uSize+63 ) >> 6 ), 0 ); } + bool Get ( uint64_t uIdx ) const { return ( ( m_dBits [ (size_t)( uIdx>>6 ) ] >> ( uIdx & 63 ) ) & 1ULL )!=0; } + void Set ( uint64_t uIdx ) { m_dBits [ (size_t)( uIdx>>6 ) ] |= 1ULL << ( uIdx & 63 ); } + +private: + std::vector m_dBits; +}; + + +struct Candidate_t +{ + float m_fDist = 0.0f; + hnswlib::tableint m_tId = INVALID_NODE; + + bool operator < ( const Candidate_t & tOther ) const + { + if ( m_fDist!=tOther.m_fDist ) + return m_fDist < tOther.m_fDist; + + return m_tId < tOther.m_tId; + } +}; + + +// Marks everything reachable from tSeed over directed level-0 links and returns how many nodes it newly marked +// When pParent is set it also records, for every newly reached node, the node whose edge reached it, i.e. a spanning tree rooted at tSeed +uint64_t MarkReachable ( const HNSWAlg_t & tAlg, hnswlib::tableint tSeed, uint64_t uCount, NodeBitmap_c & tSeen, std::vector & dStack, std::vector * pParent ) +{ + if ( tSeed>=uCount || tSeen.Get(tSeed) ) + return 0; + + tSeen.Set(tSeed); + uint64_t uMarked = 1; + + dStack.resize(0); + dStack.push_back(tSeed); + + while ( !dStack.empty() ) + { + hnswlib::tableint tCur = dStack.back(); + dStack.pop_back(); + + LinkList_t tLL = GetLinks0 ( tAlg, tCur ); + for ( uint32_t i=0; i=uCount || tSeen.Get(tNext) ) + continue; + + tSeen.Set(tNext); + if ( pParent ) + (*pParent)[tNext] = tCur; + + uMarked++; + dStack.push_back(tNext); + } + } + + return uMarked; +} + +// First candidate with a free slot; candidates are already ordered nearest-first +hnswlib::tableint FindFreeSlot ( const HNSWAlg_t & tAlg, const std::vector & dCand, uint32_t uFullThreshold ) +{ + for ( const auto & tCand : dCand ) + if ( GetLinks0 ( tAlg, tCand.m_tId ).m_uCount < uFullThreshold ) + return tCand.m_tId; + + return INVALID_NODE; +} + +// Ranks ids by distance from tOrphan and replaces dCand with the result +void RankCandidates ( const HNSWAlg_t & tAlg, hnswlib::tableint tOrphan, std::vector & dIds, std::vector & dCand, uint64_t & uDistCalls ) +{ + std::sort ( dIds.begin(), dIds.end() ); + dIds.erase ( std::unique ( dIds.begin(), dIds.end() ), dIds.end() ); + + dCand.resize(0); + dCand.reserve ( dIds.size() ); + for ( hnswlib::tableint tId : dIds ) + dCand.push_back ( { RepairDist ( tAlg, tOrphan, tId, uDistCalls ), tId } ); + + std::sort ( dCand.begin(), dCand.end() ); +} + +static thread_local uint32_t g_uRepairFullThreshold = 0; +static thread_local RepairStats_t g_tRepairStats; + +} // anonymous namespace + + +static void RepairGraphConnectivity ( HNSWAlg_t & tAlg, uint32_t uFullThreshold, RepairStats_t * pStats ) +{ + RepairStats_t tStats; + const uint64_t uCount = tAlg.cur_element_count; + tStats.m_uNodes = uCount; + + // rows with an empty vector attribute never reach addPoint, so cur_element_count can be less than max_elements_ + if ( uCount<2 ) + { + if ( pStats ) + *pStats = tStats; + + return; + } + + const hnswlib::tableint tEntry = tAlg.enterpoint_node_; + if ( tEntry>=uCount ) // -1 on an empty index + { + if ( pStats ) + *pStats = tStats; + + return; + } + + const uint32_t uMaxM0 = (uint32_t)tAlg.maxM0_; + if ( !uFullThreshold || uFullThreshold>uMaxM0 ) + uFullThreshold = uMaxM0; + + NodeBitmap_c tSeen; + tSeen.Resize(uCount); + + std::vector dStack; + dStack.reserve ( (size_t)std::min ( uCount, 1<<16 ) ); + + uint64_t uMarked = MarkReachable ( tAlg, tEntry, uCount, tSeen, dStack, nullptr ); + if ( uMarked==uCount ) // healthy index + { + if ( pStats ) + *pStats = tStats; + + return; + } + + std::vector dOrphans; + for ( uint64_t i=0; i dParent; + auto fnEnsureParents = [&] + { + if ( !dParent.empty() ) + return; + + dParent.assign ( (size_t)uCount, INVALID_NODE ); + NodeBitmap_c tTreeSeen; + tTreeSeen.Resize(uCount); + std::vector dTreeStack; + MarkReachable ( tAlg, tEntry, uCount, tTreeSeen, dTreeStack, &dParent ); + }; + + std::vector dIds; + std::vector dCand; + + for ( hnswlib::tableint tOrphan : dOrphans ) + { + if ( tSeen.Get(tOrphan) ) // already reconnected as part of an earlier orphan's component + continue; + + // 1-hop: this orphan's own reachable out-links, nearest first + LinkList_t tOrphanLinks = GetLinks0 ( tAlg, tOrphan ); + dIds.resize(0); + for ( uint32_t i=0; i=uCount ) + continue; + + LinkList_t tHop = GetLinks0 ( tAlg, tN ); + for ( uint32_t j=0; j::max(); + + for ( uint32_t i=0; i=uCount || tY==tEntry || tY==tOrphan || dParent[tY]==tCandId ) + continue; + + float fDist = RepairDist ( tAlg, tY, tCandId, tStats.m_uDistanceCalls ); // neighbour is the query + if ( fDist>fWorst ) + { + fWorst = fDist; + uVictim = i; + } + } + + if ( uVictim==UINT32_MAX ) + continue; + + tLL.m_pLinks[uVictim] = tOrphan; // count is unchanged, header untouched + tTarget = tCandId; + tStats.m_uEvictions++; + bEvicted = true; + break; + } + + if ( !bEvicted ) + continue; // best effort: leave this one alone, it is no worse than before + } + + if ( !dParent.empty() ) + dParent[tOrphan] = tTarget; + + uMarked += MarkReachable ( tAlg, tOrphan, uCount, tSeen, dStack, dParent.empty() ? nullptr : &dParent ); + } + + // Re-traverse the mutated graph from scratch + NodeBitmap_c tFinalSeen; + tFinalSeen.Resize(uCount); + std::vector dFinalStack; + uint64_t uFinalMarked = MarkReachable ( tAlg, tEntry, uCount, tFinalSeen, dFinalStack, nullptr ); + tStats.m_uUnreachableAfter = uCount - uFinalMarked; + + if ( pStats ) + *pStats = tStats; +} + + +void RepairGraphConnectivity ( HNSWAlg_t & tAlg ) +{ + RepairStats_t tStats; + RepairGraphConnectivity ( tAlg, g_uRepairFullThreshold, &tStats ); + + g_tRepairStats.m_uNodes += tStats.m_uNodes; + g_tRepairStats.m_uOrphansFound += tStats.m_uOrphansFound; + g_tRepairStats.m_uAppends += tStats.m_uAppends; + g_tRepairStats.m_u2HopWidenings += tStats.m_u2HopWidenings; + g_tRepairStats.m_uEvictions += tStats.m_uEvictions; + g_tRepairStats.m_uDistanceCalls += tStats.m_uDistanceCalls; + g_tRepairStats.m_uUnreachableAfter += tStats.m_uUnreachableAfter; + g_tRepairStats.m_uRuns++; +} + + +void Test_SetRepairFullThreshold ( uint32_t uThreshold ) +{ + g_uRepairFullThreshold = uThreshold; +} + + +void Test_ResetRepairStats() +{ + g_tRepairStats = RepairStats_t(); +} + + +const RepairStats_t & Test_GetRepairStats() +{ + return g_tRepairStats; +} + +} // namespace knn diff --git a/knn/graphrepair.h b/knn/graphrepair.h new file mode 100644 index 00000000..8f89617b --- /dev/null +++ b/knn/graphrepair.h @@ -0,0 +1,67 @@ +// Copyright (c) 2026, Manticore Software LTD (https://manticoresearch.com) +// All rights reserved +// +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file is NOT a part of the common headers (API). It is internal to the knn library and is +// not reachable through knn.h, so changes here do not require a LIB_VERSION bump. + +#pragma once + +#include "hnswlib.h" + +#include + +namespace knn +{ + +struct RepairStats_t +{ + uint64_t m_uNodes = 0; // cur_element_count on entry + uint64_t m_uOrphansFound = 0; // level-0 nodes not reachable from the entry point + uint64_t m_uAppends = 0; // relinked into a free slot + uint64_t m_u2HopWidenings = 0; // relinked via the bounded 2-hop search + uint64_t m_uEvictions = 0; // relinked by replacing a non-tree edge + uint64_t m_uDistanceCalls = 0; // distance evaluations performed by the pass + uint64_t m_uUnreachableAfter = 0; // still unreachable after a full re-traversal + uint64_t m_uRuns = 0; // how many times the pass ran (one per KNN attribute) +}; + +// Reconnects level-0 nodes that HNSW construction left with no incoming edge. +// +// hnswlib's mutuallyConnectNewElement can leave a node with zero in-edges: when a neighbour's link +// list is full it re-runs getNeighborsByHeuristic2 over the existing links plus the new node, and +// that heuristic may drop the new node outright or return fewer entries than before. +// The affected row stays in the index, but no knn() query can reach it. +// +// This pass is best effort: it never fails a build. A node it cannot relink is left exactly as it +// was, which is no worse than not running at all. +// +// MUST be called before ScalarQuantizer_i::FinalizeEncoding(). uFullThreshold is maxM0_ in production; +// a test passes a smaller value to force the eviction branch. pStats may be null. +void RepairGraphConnectivity ( hnswlib::HierarchicalNSW & tAlg ); + +/// Test seam for a build driven through the public Builder_i API, where the caller cannot reach the +/// private HNSWIndexBuilder_c::m_pAlg. All thread-local; a threshold of 0 restores the default +/// (maxM0_). +/// +/// The stats ACCUMULATE across runs and count them in m_uRuns, because one Save covers every KNN +/// attribute in the index and each attribute repairs its own graph. Reporting only the last run +/// would let a test pass while earlier attributes were skipped entirely. Call +/// Test_ResetRepairStats() before a build and check m_uRuns against the attribute count. +void Test_SetRepairFullThreshold ( uint32_t uThreshold ); +void Test_ResetRepairStats(); +const RepairStats_t & Test_GetRepairStats(); + +} // namespace knn diff --git a/knn/knn.cpp b/knn/knn.cpp index a47d7521..3769b73f 100644 --- a/knn/knn.cpp +++ b/knn/knn.cpp @@ -20,6 +20,7 @@ #include "quantizer.h" #include "termination.h" #include "space.h" +#include "graphrepair.h" #include "util/reader.h" #include "util_private.h" @@ -914,6 +915,10 @@ bool HNSWIndexBuilder_c::AddDoc ( uint32_t uRowID, const util::Span_T & d void HNSWIndexBuilder_c::Save ( FileWriter_c & tWriter ) { + // Must run before FinalizeEncoding() because the 4-bit build pool still exists at this point + if ( m_pAlg ) + RepairGraphConnectivity ( *m_pAlg ); + if ( m_pQuantizer ) m_pQuantizer->FinalizeEncoding(); diff --git a/knn/test_graphrepair.cpp b/knn/test_graphrepair.cpp new file mode 100644 index 00000000..6baa4c40 --- /dev/null +++ b/knn/test_graphrepair.cpp @@ -0,0 +1,530 @@ +// Copyright (c) 2026, Manticore Software LTD (https://manticoresearch.com) +// All rights reserved +// +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Regression test for the post-build HNSW connectivity repair pass. +// +// HNSW construction can leave a node with no incoming level-0 edge. The row stays in the index and +// is still returned by a scan, but no knn() query can reach it. The repair pass in graphrepair.cpp +// relinks such nodes before the index is written. +// + +#include "knn.h" +#include "graphrepair.h" + +#include +#include +#include +#include +#include + +static int g_iFailures = 0; + +#define CHECK( _expr, ... ) \ + do { if (!(_expr)) { printf ( "FAILED %s:%d: %s\n ", __FILE__, __LINE__, #_expr ); printf ( __VA_ARGS__ ); printf ( "\n" ); g_iFailures++; } } while(0) + + +// Deterministic fixture. Tight clusters plus a few far outliers, because outliers are what the +// construction heuristic gives 1-2 out-links to, and those are the nodes that end up stranded +class LCG_c +{ +public: + explicit LCG_c ( uint64_t uSeed ) : m_uState ( uSeed ) {} + + uint64_t Next() + { + m_uState = m_uState*6364136223846793005ULL + 1442695040888963407ULL; + return m_uState; + } + + double Unit() { return double ( Next()>>11 ) * ( 1.0/9007199254740992.0 ); } + + double Gauss() + { + double fRes = 0.0; + for ( int i=0; i<12; i++ ) + fRes += Unit(); + + return fRes-6.0; + } + +private: + uint64_t m_uState; +}; + + +static std::vector GenerateFixture ( int iRows, int iDims, int iClusters, int iOutlierPct, uint64_t uSeed ) +{ + LCG_c tRng(uSeed); + + std::vector> dCentroids ( iClusters, std::vector(iDims) ); + for ( auto & dCentroid : dCentroids ) + for ( auto & fVal : dCentroid ) + fVal = tRng.Gauss(); + + std::vector dData ( (size_t)iRows*iDims ); + std::vector dPoint(iDims); + for ( int i=0; i & dData, int iRows, int iDims, knn::HNSWSimilarity_e eSimilarity, knn::Quantization_e eQuantization, const std::string & sTag, uint32_t uFullThreshold, int iHNSWM = 16, int iEFConstruction = 200, int iNumElements = -1 ) +{ + BuildResult_t tRes; + tRes.m_sFilename = "graphrepair_test_" + sTag + ".spknn"; + if ( iNumElements<0 ) + iNumElements = iRows; + + knn::AttrWithSettings_t tAttr; + tAttr.m_sName = "vec"; + tAttr.m_bKNN = true; + tAttr.m_iDims = iDims; + tAttr.m_eHNSWSimilarity = eSimilarity; + tAttr.m_eQuantization = eQuantization; + tAttr.m_iHNSWM = iHNSWM; + tAttr.m_iHNSWEFConstruction = iEFConstruction; + + knn::Schema_t dSchema { tAttr }; + std::unique_ptr pBuilder { CreateKNNBuilder ( dSchema, iNumElements, tRes.m_sFilename+".tmp" ) }; + if ( !pBuilder ) + { + printf ( "FAILED %s: CreateKNNBuilder returned null\n", sTag.c_str() ); + g_iFailures++; + return tRes; + } + + for ( int i=0; iTrain ( 0, i, { (float*)dData.data() + (size_t)i*iDims, (size_t)iDims } ); + + std::string sError; + if ( !pBuilder->FinalizeTraining(sError) ) + { + printf ( "FAILED %s: FinalizeTraining: %s\n", sTag.c_str(), sError.c_str() ); + g_iFailures++; + return tRes; + } + + knn::BuildContext_t tBuildCtx; + for ( int i=0; iSetAttr ( 0, i, { (float*)dData.data() + (size_t)i*iDims, (size_t)iDims }, tBuildCtx ) ) + { + printf ( "FAILED %s: SetAttr row %d: %s\n", sTag.c_str(), i, tBuildCtx.m_sError.c_str() ); + g_iFailures++; + return tRes; + } + + knn::Test_SetRepairFullThreshold(uFullThreshold); + knn::Test_ResetRepairStats(); + bool bSaved = pBuilder->Save ( tRes.m_sFilename, 1024*1024, sError ); + knn::Test_SetRepairFullThreshold(0); + + tRes.m_tStats = knn::Test_GetRepairStats(); + if ( !bSaved ) + { + printf ( "FAILED %s: Save: %s\n", sTag.c_str(), sError.c_str() ); + g_iFailures++; + return tRes; + } + + tRes.m_bOk = true; + return tRes; +} + + +// Every row must be findable by searching with its own vector +static int CountUnfindable ( const std::string & sFilename, const std::vector & dData, int iRows, int iDims, int iK ) +{ + std::unique_ptr pKNN { CreateKNN() }; + std::string sError; + if ( !pKNN->Load ( sFilename, sError ) ) + { + printf ( "FAILED: KNN load '%s': %s\n", sFilename.c_str(), sError.c_str() ); + g_iFailures++; + return -1; + } + + int iUnfindable = 0; + for ( int i=0; i dPoint { (float*)dData.data() + (size_t)i*iDims, (size_t)iDims }; + std::unique_ptr pIt { pKNN->CreateIterator ( "vec", dPoint, iK, 128, nullptr, knn::HNSWTerminationPolicy_e::NONE, false, sError ) }; + if ( !pIt ) + { + printf ( "FAILED: CreateIterator row %d: %s\n", i, sError.c_str() ); + g_iFailures++; + return -1; + } + + bool bFound = false; + for ( const auto & tHit : pIt->GetData() ) + if ( tHit.m_tRowID==(uint32_t)i ) + { + bFound = true; + break; + } + + if ( !bFound ) + iUnfindable++; + } + + return iUnfindable; +} + + +static bool BuildMultiAttr ( const std::vector & dData, int iRows, int iDims, int iAttrs, const char * szTag, knn::RepairStats_t & tLast ) +{ + knn::Schema_t dSchema; + for ( int i=0; i pBuilder { CreateKNNBuilder ( dSchema, iRows, sFilename+".tmp" ) }; + if ( !pBuilder ) + return false; + + for ( int iAttr=0; iAttrTrain ( iAttr, i, { (float*)dData.data() + (size_t)i*iDims, (size_t)iDims } ); + + std::string sError; + if ( !pBuilder->FinalizeTraining(sError) ) + return false; + + knn::BuildContext_t tBuildCtx; + for ( int iAttr=0; iAttrSetAttr ( iAttr, i, { (float*)dData.data() + (size_t)i*iDims, (size_t)iDims }, tBuildCtx ) ) + return false; + + knn::Test_ResetRepairStats(); + bool bOk = pBuilder->Save ( sFilename, 1024*1024, sError ); + tLast = knn::Test_GetRepairStats(); // accumulated over every attribute, with m_uRuns + return bOk; +} + + +static bool BuildMultiVector ( const std::vector & dData, int iRows, int iDims, int iVecsPerRow, knn::RepairStats_t & tLast ) +{ + knn::AttrWithSettings_t tAttr; + tAttr.m_sName = "vec"; + tAttr.m_bKNN = true; + tAttr.m_iDims = iDims; + tAttr.m_eHNSWSimilarity = knn::HNSWSimilarity_e::COSINE; + tAttr.m_iHNSWM = 16; + tAttr.m_iHNSWEFConstruction = 200; + tAttr.m_bMulti = true; + + const int iDocs = iRows/iVecsPerRow; + knn::Schema_t dSchema { tAttr }; + std::string sFilename = "graphrepair_test_multivec.spknn"; + std::unique_ptr pBuilder { CreateKNNBuilder ( dSchema, iDocs, sFilename+".tmp" ) }; + if ( !pBuilder ) + return false; + + const size_t uRowFloats = (size_t)iDims*iVecsPerRow; + for ( int i=0; iTrain ( 0, i, { (float*)dData.data() + (size_t)i*uRowFloats, uRowFloats } ); + + std::string sError; + if ( !pBuilder->FinalizeTraining(sError) ) + return false; + + knn::BuildContext_t tBuildCtx; + for ( int i=0; iSetAttr ( 0, i, { (float*)dData.data() + (size_t)i*uRowFloats, uRowFloats }, tBuildCtx ) ) + return false; + + knn::Test_ResetRepairStats(); + bool bOk = pBuilder->Save ( sFilename, 1024*1024, sError ); + tLast = knn::Test_GetRepairStats(); + return bOk; +} + + +/// Fixture search. Fixture needs regenerating if hardcoded settings no longer produce orphans +static int RunSweep() +{ + struct Mode_t { knn::HNSWSimilarity_e m_eSim; const char * m_szSim; knn::Quantization_e m_eQuant; const char * m_szQuant; }; + const Mode_t dModes[] = + { + { knn::HNSWSimilarity_e::COSINE, "cosine", knn::Quantization_e::NONE, "none" }, + { knn::HNSWSimilarity_e::COSINE, "cosine", knn::Quantization_e::BIT8, "8bit" }, + { knn::HNSWSimilarity_e::COSINE, "cosine", knn::Quantization_e::BIT1, "1bit" }, + { knn::HNSWSimilarity_e::L2, "l2", knn::Quantization_e::BIT1, "1bit" }, + }; + + const int dM[] = { 4, 8, 16 }; + const int dOutliers[] = { 3, 10 }; + + for ( const auto & tMode : dModes ) + for ( int iHNSWM : dM ) + for ( int iOutlierPct : dOutliers ) + for ( uint64_t uSeed=1; uSeed<=2; uSeed++ ) + { + const int iRows = 5000, iDims = 64; + std::vector dData = GenerateFixture ( iRows, iDims, 8, iOutlierPct, uSeed ); + BuildResult_t tRes = BuildIndex ( dData, iRows, iDims, tMode.m_eSim, tMode.m_eQuant, "sweep", 0, iHNSWM ); + printf ( "%-6s %-4s M=%-3d outlier%%=%-3d seed=%llu -> orphans=%llu appends=%llu widenings=%llu unreachable_after=%llu\n", + tMode.m_szSim, tMode.m_szQuant, iHNSWM, iOutlierPct, (unsigned long long)uSeed, + (unsigned long long)tRes.m_tStats.m_uOrphansFound, (unsigned long long)tRes.m_tStats.m_uAppends, + (unsigned long long)tRes.m_tStats.m_u2HopWidenings, + (unsigned long long)tRes.m_tStats.m_uUnreachableAfter ); + } + + return 0; +} + + +// Reports how many nodes a build would strand for vectors loaded from a text file (" " then row-major floats) +static int RunOrphanCount ( const char * szFile, int iHNSWM ) +{ + FILE * pFile = fopen ( szFile, "rt" ); + if ( !pFile ) + { + printf ( "cannot open '%s'\n", szFile ); + return 1; + } + + int iRows = 0, iDims = 0; + if ( fscanf ( pFile, "%d %d", &iRows, &iDims )!=2 || iRows<=0 || iDims<=0 ) + { + printf ( "bad header in '%s'\n", szFile ); + fclose(pFile); + return 1; + } + + std::vector dData ( (size_t)iRows*iDims ); + for ( auto & fVal : dData ) + if ( fscanf ( pFile, "%f", &fVal )!=1 ) + { + printf ( "truncated data in '%s'\n", szFile ); + fclose(pFile); + return 1; + } + + fclose(pFile); + + BuildResult_t tRes = BuildIndex ( dData, iRows, iDims, knn::HNSWSimilarity_e::COSINE, knn::Quantization_e::NONE, "orphancount", 0, iHNSWM ); + printf ( "rows=%d dims=%d M=%d -> orphans=%llu appends=%llu unreachable_after=%llu\n", iRows, iDims, iHNSWM, + (unsigned long long)tRes.m_tStats.m_uOrphansFound, (unsigned long long)tRes.m_tStats.m_uAppends, + (unsigned long long)tRes.m_tStats.m_uUnreachableAfter ); + + return 0; +} + + +int main ( int iArgs, char ** ppArgs ) +{ + if ( iArgs>1 && std::string(ppArgs[1])=="--sweep" ) + return RunSweep(); + + if ( iArgs>2 && std::string(ppArgs[1])=="--orphan-count" ) + return RunOrphanCount ( ppArgs[2], iArgs>3 ? atoi(ppArgs[3]) : 16 ); + + const int iRows = 5000, iDims = 64; + std::vector dData = GenerateFixture ( iRows, iDims, 8, 3, 1 ); + + // 1. The fixture must actually contain orphans, otherwise everything test don't test anything + printf ( "--- baseline: cosine / no quantization ---\n" ); + BuildResult_t tBase = BuildIndex ( dData, iRows, iDims, knn::HNSWSimilarity_e::COSINE, knn::Quantization_e::NONE, "cosine_none", 0 ); + printf ( "nodes=%llu orphans=%llu appends=%llu widenings=%llu evictions=%llu distcalls=%llu unreachable_after=%llu\n", + (unsigned long long)tBase.m_tStats.m_uNodes, (unsigned long long)tBase.m_tStats.m_uOrphansFound, + (unsigned long long)tBase.m_tStats.m_uAppends, (unsigned long long)tBase.m_tStats.m_u2HopWidenings, + (unsigned long long)tBase.m_tStats.m_uEvictions, (unsigned long long)tBase.m_tStats.m_uDistanceCalls, + (unsigned long long)tBase.m_tStats.m_uUnreachableAfter ); + + CHECK ( tBase.m_tStats.m_uNodes==(uint64_t)iRows, "expected %d nodes", iRows ); + CHECK ( tBase.m_tStats.m_uOrphansFound>0, "fixture produced no orphans, regenerate it" ); + CHECK ( tBase.m_tStats.m_uAppends>0, "no node was relinked" ); + CHECK ( tBase.m_tStats.m_uDistanceCalls>0, "repair evaluated no distances" ); + CHECK ( tBase.m_tStats.m_uUnreachableAfter==0, "%llu nodes still unreachable", (unsigned long long)tBase.m_tStats.m_uUnreachableAfter ); + + if ( tBase.m_bOk ) + { + int iUnfindable = CountUnfindable ( tBase.m_sFilename, dData, iRows, iDims, 5 ); + printf ( "rows not findable by their own vector: %d\n", iUnfindable ); + CHECK ( iUnfindable==0, "%d rows cannot be found by knn()", iUnfindable ); + } + + // 2. Quantization matrix. M is part of the fixture - 1-bit needs M<=8 here to orphan at all + printf ( "\n--- similarity x quantization matrix ---\n" ); + struct Case_t + { + knn::HNSWSimilarity_e m_eSim; + const char * m_szSim; + knn::Quantization_e m_eQuant; + const char * m_szQuant; + int m_iM; + bool m_bMustOrphan; + }; + + const Case_t dCases[] = + { + { knn::HNSWSimilarity_e::L2, "l2", knn::Quantization_e::NONE, "none", 16, true }, + { knn::HNSWSimilarity_e::IP, "ip", knn::Quantization_e::NONE, "none", 16, true }, + { knn::HNSWSimilarity_e::COSINE, "cosine", knn::Quantization_e::BIT8, "8bit", 16, true }, + { knn::HNSWSimilarity_e::L2, "l2", knn::Quantization_e::BIT8, "8bit", 8, true }, + { knn::HNSWSimilarity_e::COSINE, "cosine", knn::Quantization_e::BIT1, "1bit", 8, true }, + { knn::HNSWSimilarity_e::L2, "l2", knn::Quantization_e::BIT1, "1bit", 8, true }, + { knn::HNSWSimilarity_e::COSINE, "cosine", knn::Quantization_e::BIT1, "1bit", 4, true }, + { knn::HNSWSimilarity_e::IP, "ip", knn::Quantization_e::BIT8, "8bit", 16, true }, + { knn::HNSWSimilarity_e::IP, "ip", knn::Quantization_e::BIT1, "1bit", 8, true }, + { knn::HNSWSimilarity_e::COSINE, "cosine", knn::Quantization_e::BIT1, "1bit", 16, false }, + }; + + for ( const auto & tCase : dCases ) + { + std::string sTag = std::string(tCase.m_szSim) + "_" + tCase.m_szQuant + "_m" + std::to_string(tCase.m_iM); + BuildResult_t tRes = BuildIndex ( dData, iRows, iDims, tCase.m_eSim, tCase.m_eQuant, sTag, 0, tCase.m_iM ); + printf ( "%-20s orphans=%-5llu appends=%-5llu evictions=%-3llu distcalls=%-6llu unreachable_after=%llu\n", sTag.c_str(), + (unsigned long long)tRes.m_tStats.m_uOrphansFound, (unsigned long long)tRes.m_tStats.m_uAppends, + (unsigned long long)tRes.m_tStats.m_uEvictions, (unsigned long long)tRes.m_tStats.m_uDistanceCalls, + (unsigned long long)tRes.m_tStats.m_uUnreachableAfter ); + + CHECK ( tRes.m_bOk, "%s: build failed", sTag.c_str() ); + CHECK ( tRes.m_tStats.m_uUnreachableAfter==0, "%s: %llu nodes still unreachable", sTag.c_str(), (unsigned long long)tRes.m_tStats.m_uUnreachableAfter ); + + if ( tCase.m_bMustOrphan ) + { + CHECK ( tRes.m_tStats.m_uOrphansFound>0, "%s: fixture no longer strands anything, re-pin it with --sweep", sTag.c_str() ); + CHECK ( tRes.m_tStats.m_uAppends>0, "%s: nothing was relinked", sTag.c_str() ); + CHECK ( tRes.m_tStats.m_uDistanceCalls>0, "%s: repair evaluated no distances, so this case never exercises the build kernel", sTag.c_str() ); + } + + // Only meaningful without quantization. A quantized search compares quantized codes, so a self-query with the original float vector can legitimately miss the row + if ( tRes.m_bOk && tCase.m_eQuant==knn::Quantization_e::NONE ) + { + int iUnfindable = CountUnfindable ( tRes.m_sFilename, dData, iRows, iDims, 20 ); + CHECK ( iUnfindable==0, "%s: %d rows cannot be found by knn()", sTag.c_str(), iUnfindable ); + } + } + + // 3. Force the eviction branch. Real data never fills enough lists to reach it, so lower the + // "list is full" threshold. Eviction must relink without stranding anything. + printf ( "\n--- forced eviction (full threshold = 4) ---\n" ); + BuildResult_t tEvict = BuildIndex ( dData, iRows, iDims, knn::HNSWSimilarity_e::COSINE, knn::Quantization_e::NONE, "evict", 4 ); + printf ( "orphans=%llu appends=%llu evictions=%llu distcalls=%llu unreachable_after=%llu\n", + (unsigned long long)tEvict.m_tStats.m_uOrphansFound, (unsigned long long)tEvict.m_tStats.m_uAppends, + (unsigned long long)tEvict.m_tStats.m_uEvictions, (unsigned long long)tEvict.m_tStats.m_uDistanceCalls, + (unsigned long long)tEvict.m_tStats.m_uUnreachableAfter ); + + CHECK ( tEvict.m_tStats.m_uEvictions>0, "eviction branch never ran, so it is untested" ); + CHECK ( tEvict.m_tStats.m_uDistanceCalls>0, "eviction ranked no victims by distance" ); + CHECK ( tEvict.m_tStats.m_uUnreachableAfter==0, "eviction stranded %llu nodes", (unsigned long long)tEvict.m_tStats.m_uUnreachableAfter ); + + if ( tEvict.m_bOk ) + { + int iUnfindable = CountUnfindable ( tEvict.m_sFilename, dData, iRows, iDims, 5 ); + printf ( "rows not findable by their own vector: %d\n", iUnfindable ); + CHECK ( iUnfindable==0, "%d rows cannot be found by knn() after eviction", iUnfindable ); + } + + // 4. Degenerate sizes must not trip the entry-point guards. Zero elements matters most: the graph + // is empty and enterpoint_node_ is (tableint)-1, so the pass has to bail before touching it. + printf ( "\n--- degenerate sizes ---\n" ); + for ( int iSmall : { 0, 1, 2, 3 } ) + { + std::vector dSmall ( dData.begin(), dData.begin() + (size_t)iSmall*iDims ); + BuildResult_t tRes = BuildIndex ( dSmall, iSmall, iDims, knn::HNSWSimilarity_e::COSINE, knn::Quantization_e::NONE, "small" + std::to_string(iSmall), 0 ); + printf ( "rows=%-2d -> ok=%d nodes=%llu unreachable_after=%llu\n", iSmall, (int)tRes.m_bOk, + (unsigned long long)tRes.m_tStats.m_uNodes, (unsigned long long)tRes.m_tStats.m_uUnreachableAfter ); + CHECK ( tRes.m_bOk, "%d-row build failed", iSmall ); + CHECK ( tRes.m_tStats.m_uNodes==(uint64_t)iSmall, "%d-row build reported %llu nodes", iSmall, (unsigned long long)tRes.m_tStats.m_uNodes ); + CHECK ( tRes.m_tStats.m_uUnreachableAfter==0, "%d-row index has unreachable nodes", iSmall ); + } + + // 5. cur_element_count < max_elements_. Rows whose vector attribute is empty never reach addPoint, so the graph is smaller than the capacity it was built with. + // Iterating to max_elements_ would walk uninitialised level-0 memory. + printf ( "\n--- cur_element_count < max_elements_ ---\n" ); + { + const int iPartial = iRows/2; + std::vector dPartial ( dData.begin(), dData.begin() + (size_t)iPartial*iDims ); + BuildResult_t tRes = BuildIndex ( dPartial, iPartial, iDims, knn::HNSWSimilarity_e::COSINE, knn::Quantization_e::NONE, "partial", 0, 16, 200, iRows ); + printf ( "capacity=%d filled=%d -> nodes=%llu orphans=%llu unreachable_after=%llu\n", iRows, iPartial, + (unsigned long long)tRes.m_tStats.m_uNodes, (unsigned long long)tRes.m_tStats.m_uOrphansFound, + (unsigned long long)tRes.m_tStats.m_uUnreachableAfter ); + CHECK ( tRes.m_bOk, "partially filled build failed" ); + CHECK ( tRes.m_tStats.m_uNodes==(uint64_t)iPartial, "pass saw %llu nodes, expected the filled count %d", (unsigned long long)tRes.m_tStats.m_uNodes, iPartial ); + CHECK ( tRes.m_tStats.m_uUnreachableAfter==0, "partially filled index has unreachable nodes" ); + } + + // 6. Several KNN attributes on one index: each owns its own graph and its own Save call. + printf ( "\n--- multiple KNN attributes ---\n" ); + { + const int iAttrs = 3; + knn::RepairStats_t tAcc; + bool bOk = BuildMultiAttr ( dData, iRows, iDims, iAttrs, "multiattr", tAcc ); + printf ( "%d attributes -> ok=%d runs=%llu nodes(sum)=%llu orphans(sum)=%llu unreachable_after(sum)=%llu\n", + iAttrs, (int)bOk, (unsigned long long)tAcc.m_uRuns, (unsigned long long)tAcc.m_uNodes, + (unsigned long long)tAcc.m_uOrphansFound, (unsigned long long)tAcc.m_uUnreachableAfter ); + CHECK ( bOk, "multi-attribute build failed" ); + // the stats accumulate, so this catches an attribute being skipped entirely - reading only the + // last run would pass even if the earlier graphs were never repaired + CHECK ( tAcc.m_uRuns==(uint64_t)iAttrs, "repair ran %llu times for %d attributes", (unsigned long long)tAcc.m_uRuns, iAttrs ); + CHECK ( tAcc.m_uNodes==(uint64_t)iRows*iAttrs, "multi-attribute: pass saw %llu nodes across all attributes, expected %d", (unsigned long long)tAcc.m_uNodes, iRows*iAttrs ); + CHECK ( tAcc.m_uOrphansFound>0, "multi-attribute fixture stranded nothing, so it proves nothing" ); + CHECK ( tAcc.m_uUnreachableAfter==0, "multi-attribute index has unreachable nodes" ); + } + + // 7. Multi-vector attribute: labels are vector ids, and the graph is allocated in FinalizeTraining rather than the constructor. + printf ( "\n--- multi-vector attribute ---\n" ); + { + knn::RepairStats_t tAcc; + bool bOk = BuildMultiVector ( dData, iRows, iDims, 2, tAcc ); + printf ( "2 vectors/row -> ok=%d runs=%llu nodes=%llu orphans=%llu unreachable_after=%llu\n", (int)bOk, + (unsigned long long)tAcc.m_uRuns, (unsigned long long)tAcc.m_uNodes, + (unsigned long long)tAcc.m_uOrphansFound, (unsigned long long)tAcc.m_uUnreachableAfter ); + CHECK ( bOk, "multi-vector build failed" ); + CHECK ( tAcc.m_uRuns==1, "repair ran %llu times for one attribute", (unsigned long long)tAcc.m_uRuns ); + CHECK ( tAcc.m_uNodes==(uint64_t)iRows, "multi-vector: pass saw %llu nodes, expected %d vectors", (unsigned long long)tAcc.m_uNodes, iRows ); + CHECK ( tAcc.m_uUnreachableAfter==0, "multi-vector index has unreachable nodes" ); + } + + printf ( "\n%s (%d failure(s))\n", g_iFailures ? "FAILED" : "PASSED", g_iFailures ); + return g_iFailures ? 1 : 0; +}