fix: EmbList serialization compatibility across index versions [3.0]

include/knowhere/index/emb_list_strategy.h

@@ -279,6 +279,12 @@ class EmbListStrategy {
 
 using EmbListStrategyPtr = std::unique_ptr<EmbListStrategy>;
 
+// Index versions before 11 use the legacy TokenANN EMB_LIST_META format.  Version 11
+// introduces the strategy-aware format below.  Keep the reader magic-based because
+// historical indexes may have been serialized with either format for the same index
+// version.
+constexpr IndexVersion kEmbListMetaV2MinVersion = 11;
+
 // Magic number for EMB_LIST_META format: [magic][type_len][type][strategy_blob]
 // Use int64_t to avoid collision with legacy TokenANN format whose first 8 bytes are size_t count.
 constexpr int64_t kEmbListMetaMagic = 0x454C4D465F563200LL;  // "ELMF_V2\0"

include/knowhere/index/index_node.h

@@ -586,6 +586,16 @@ class IndexNode : public Object {
     Status
     SerializeEmbList(BinarySet& binset) const;
 
+    /**
+     * @brief Serialize the EMB_LIST_META payload according to the target index version.
+     *
+     * Versions before kEmbListMetaV2MinVersion use the legacy TokenANN-only
+     * [size_t count][size_t[count] offsets] format. Newer versions use the
+     * strategy-aware [magic][type_len][type][strategy_blob] format.
+     */
+    Status
+    SerializeEmbListMeta(std::shared_ptr<uint8_t[]>& data, int64_t& size) const;
+
     /**
      * @brief Deserialize emb_list: base index, strategy, raw index, and ID mapping from BinarySet.
      */

src/index/hnsw/faiss_hnsw.cc

@@ -70,7 +70,7 @@ namespace knowhere {
 //
 class BaseFaissIndexNode : public IndexNode {
  public:
-    BaseFaissIndexNode(const int32_t& /*version*/, const Object& object) {
+    BaseFaissIndexNode(const int32_t& version, const Object& object) : IndexNode(version) {
         build_pool = ThreadPool::GetGlobalBuildThreadPool();
         search_pool = ThreadPool::GetGlobalSearchThreadPool();
     }
@@ -2190,7 +2190,7 @@ class BaseFaissRegularIndexHNSWFlatNodeTemplate : public BaseFaissRegularIndexHN
 //   FallbackSearchIndex properly.
 class HNSWIndexNodeWithFallback : public IndexNode {
  public:
-    HNSWIndexNodeWithFallback(const int32_t& version, const Object& object) {
+    HNSWIndexNodeWithFallback(const int32_t& version, const Object& object) : IndexNode(version) {
         constexpr int faiss_hnsw_support_version = 6;
         if (version >= faiss_hnsw_support_version) {
             use_base_index = true;

src/index/hnsw/hnsw.h

@@ -38,7 +38,7 @@ template <typename DataType, QuantType quant_type = QuantType::None>
 class HnswIndexNode : public IndexNode {
  public:
     using DistType = float;
-    HnswIndexNode(const int32_t& /*version*/, const Object& object) : index_(nullptr) {
+    HnswIndexNode(const int32_t& version, const Object& object) : IndexNode(version), index_(nullptr) {
         search_pool_ = ThreadPool::GetGlobalSearchThreadPool();
     }
 

src/index/index_node.cc

@@ -513,8 +513,29 @@ IndexNode::ParseEmbListMetaHeader(const uint8_t* data, int64_t size) {
 }
 
 Status
-IndexNode::SerializeEmbList(BinarySet& binset) const {
-    LOG_KNOWHERE_INFO_ << "Serialize emb_list with strategy: " << emb_list_strategy_->Type();
+IndexNode::SerializeEmbListMeta(std::shared_ptr<uint8_t[]>& data, int64_t& size) const {
+    if (!emb_list_strategy_) {
+        return Status::emb_list_inner_error;
+    }
+
+    if (version_.VersionNumber() < kEmbListMetaV2MinVersion) {
+        if (emb_list_strategy_->Type() != meta::EMB_LIST_STRATEGY_TOKENANN) {
+            LOG_KNOWHERE_WARNING_ << "Legacy emb_list meta only supports TokenANN, got: " << emb_list_strategy_->Type()
+                                  << ", index version: " << version_.VersionNumber();
+            return Status::not_implemented;
+        }
+
+        auto emb_list_offset = emb_list_strategy_->GetEmbListOffset();
+        if (!emb_list_offset) {
+            return Status::emb_list_inner_error;
+        }
+
+        size = static_cast<int64_t>(EmbListOffsetByteSize(emb_list_offset));
+        data = std::shared_ptr<uint8_t[]>(new uint8_t[size]);
+        SerializeEmbListOffsetToBytes(emb_list_offset, data.get());
+        return Status::success;
+    }
+
     try {
         // 1. Get strategy blob
         std::shared_ptr<uint8_t[]> strategy_data;
@@ -532,8 +553,24 @@ IndexNode::SerializeEmbList(BinarySet& binset) const {
         writer(strategy_type.data(), type_len, 1);
         writer(strategy_data.get(), strategy_size, 1);
 
-        std::shared_ptr<uint8_t[]> meta_data(writer.data());
-        binset.Append(meta::EMB_LIST_META, meta_data, writer.tellg());
+        data = std::shared_ptr<uint8_t[]>(writer.data());
+        size = writer.tellg();
+        return Status::success;
+    } catch (const std::exception& e) {
+        LOG_KNOWHERE_WARNING_ << "serialize emb_list meta error: " << e.what();
+        return Status::emb_list_inner_error;
+    }
+}
+
+Status
+IndexNode::SerializeEmbList(BinarySet& binset) const {
+    LOG_KNOWHERE_INFO_ << "Serialize emb_list with strategy: " << emb_list_strategy_->Type()
+                       << ", index version: " << version_.VersionNumber();
+    try {
+        std::shared_ptr<uint8_t[]> meta_data;
+        int64_t meta_size = 0;
+        RETURN_IF_ERROR(SerializeEmbListMeta(meta_data, meta_size));
+        binset.Append(meta::EMB_LIST_META, meta_data, meta_size);
 
         // 3. Raw vector index as separate key (large, needs mmap in file path)
         if (emb_list_raw_index_) {

tests/ut/test_emb_list.cc

@@ -2541,6 +2541,81 @@ TEST_CASE("Test brute force anniterator on chunk", "[on_chunk]") {
     }
 }
 
+TEST_CASE("EmbList serialization format by index version", "[emb_list][serialization][version]") {
+    const int32_t DIM = 4;
+    const int32_t NB = 64;
+    const int32_t EACH_EL_LEN = 8;
+
+    auto dataset = GenEmbListDataSet(NB, DIM, 42, EACH_EL_LEN);
+
+    knowhere::Json base_conf;
+    base_conf[knowhere::indexparam::HNSW_M] = 16;
+    base_conf[knowhere::indexparam::EFCONSTRUCTION] = 96;
+    base_conf[knowhere::meta::DIM] = DIM;
+    base_conf[knowhere::meta::ROWS] = NB;
+    base_conf[knowhere::meta::INDEX_TYPE] = knowhere::IndexEnum::INDEX_HNSW;
+    base_conf[knowhere::meta::METRIC_TYPE] = "MAX_SIM_IP";
+
+    SECTION("TokenANN uses legacy meta before V2 minimum version") {
+        constexpr auto version = knowhere::kEmbListMetaV2MinVersion - 1;
+        auto index =
+            knowhere::IndexFactory::Instance().Create<knowhere::fp32>(knowhere::IndexEnum::INDEX_HNSW, version).value();
+        REQUIRE(index.Build(dataset, base_conf) == knowhere::Status::success);
+
+        knowhere::BinarySet binset;
+        REQUIRE(index.Serialize(binset) == knowhere::Status::success);
+
+        auto meta_bin = binset.GetByName(knowhere::meta::EMB_LIST_META);
+        REQUIRE(meta_bin != nullptr);
+
+        size_t count = 0;
+        std::memcpy(&count, meta_bin->data.get(), sizeof(count));
+        REQUIRE(count == static_cast<size_t>(NB / EACH_EL_LEN + 1));
+        REQUIRE(meta_bin->size == static_cast<int64_t>(sizeof(size_t) + count * sizeof(size_t)));
+
+        int64_t first_bytes = 0;
+        std::memcpy(&first_bytes, meta_bin->data.get(), sizeof(first_bytes));
+        REQUIRE(first_bytes != knowhere::kEmbListMetaMagic);
+
+        auto loaded_index =
+            knowhere::IndexFactory::Instance().Create<knowhere::fp32>(knowhere::IndexEnum::INDEX_HNSW, version).value();
+        REQUIRE(loaded_index.Deserialize(binset, base_conf) == knowhere::Status::success);
+    }
+
+    SECTION("TokenANN uses V2 meta from V2 minimum version") {
+        constexpr auto version = knowhere::kEmbListMetaV2MinVersion;
+        auto index =
+            knowhere::IndexFactory::Instance().Create<knowhere::fp32>(knowhere::IndexEnum::INDEX_HNSW, version).value();
+        REQUIRE(index.Build(dataset, base_conf) == knowhere::Status::success);
+
+        knowhere::BinarySet binset;
+        REQUIRE(index.Serialize(binset) == knowhere::Status::success);
+
+        auto meta_bin = binset.GetByName(knowhere::meta::EMB_LIST_META);
+        REQUIRE(meta_bin != nullptr);
+
+        int64_t magic = 0;
+        std::memcpy(&magic, meta_bin->data.get(), sizeof(magic));
+        REQUIRE(magic == knowhere::kEmbListMetaMagic);
+    }
+
+    SECTION("Non-TokenANN strategy is not serializable with legacy index version") {
+        constexpr auto version = knowhere::kEmbListMetaV2MinVersion - 1;
+        auto conf = base_conf;
+        conf["emb_list_strategy"] = "muvera";
+        conf["muvera_num_projections"] = 3;
+        conf["muvera_num_repeats"] = 2;
+        conf["muvera_seed"] = 42;
+
+        auto index =
+            knowhere::IndexFactory::Instance().Create<knowhere::fp32>(knowhere::IndexEnum::INDEX_HNSW, version).value();
+        REQUIRE(index.Build(dataset, conf) == knowhere::Status::success);
+
+        knowhere::BinarySet binset;
+        REQUIRE(index.Serialize(binset) == knowhere::Status::not_implemented);
+    }
+}
+
 TEST_CASE("EmbList Serialization", "Strategy and IndexNode serialization/deserialization tests") {
     const int32_t DIM = 4;
     const int32_t NB = 64;