add NGT index (#3555)

* add NGT index
Signed-off-by: Nfenglv <fenglv15@mails.ucas.ac.cn>

NOTICE.md

@@ -23,3 +23,4 @@
 | hnswlib          | [Apache 2.0](https://github.com/nmslib/hnswlib/blob/master/LICENSE)                                                            |
 | annoy            | [Apache 2.0](https://github.com/spotify/annoy/blob/master/LICENSE)                                                             |
 | crc32c           | [BSD 3-Clause](https://github.com/google/crc32c/blob/master/LICENSE)                                                           |
+| NGT              | [Apache 2.0](https://github.com/yahoojapan/NGT/blob/master/LICENSE)                                                            |

core/src/index/cmake/ThirdPartyPackagesCore.cmake

@@ -653,3 +653,5 @@ if (KNOWHERE_WITH_FAISS AND NOT TARGET faiss_ep)
     include_directories(SYSTEM "${FAISS_INCLUDE_DIR}")
     link_directories(SYSTEM ${FAISS_PREFIX}/lib/)
 endif ()
+
+add_subdirectory(thirdparty/NGT)

core/src/index/knowhere/CMakeLists.txt

@@ -13,6 +13,7 @@
 
 include_directories(${INDEX_SOURCE_DIR}/knowhere)
 include_directories(${INDEX_SOURCE_DIR}/thirdparty)
+include_directories(${INDEX_SOURCE_DIR}/thirdparty/NGT/lib)
 
 if (MILVUS_SUPPORT_SPTAG)
     include_directories(${INDEX_SOURCE_DIR}/thirdparty/SPTAG/AnnService)
@@ -68,6 +69,9 @@ set(vector_index_srcs
         knowhere/index/vector_index/IndexRHNSWFlat.cpp
         knowhere/index/vector_index/IndexRHNSWSQ.cpp
         knowhere/index/vector_index/IndexRHNSWPQ.cpp
+        knowhere/index/vector_index/IndexNGT.cpp
+        knowhere/index/vector_index/IndexNGTPANNG.cpp
+        knowhere/index/vector_index/IndexNGTONNG.cpp
         )
 
 set(vector_offset_index_srcs
@@ -91,6 +95,7 @@ set(depend_libs
         gfortran
         pthread
         fiu
+        ngt
         )
 
 if (MILVUS_SUPPORT_SPTAG)

core/src/index/knowhere/knowhere/index/IndexType.cpp

@@ -37,6 +37,8 @@ const char* INDEX_RHNSWFlat = "RHNSW_FLAT";
 const char* INDEX_RHNSWPQ = "RHNSW_PQ";
 const char* INDEX_RHNSWSQ = "RHNSW_SQ";
 const char* INDEX_ANNOY = "ANNOY";
+const char* INDEX_NGTPANNG = "NGT_PANNG";
+const char* INDEX_NGTONNG = "NGT_ONNG";
 }  // namespace IndexEnum
 
 }  // namespace knowhere

core/src/index/knowhere/knowhere/index/IndexType.h

@@ -64,6 +64,8 @@ extern const char* INDEX_RHNSWFlat;
 extern const char* INDEX_RHNSWPQ;
 extern const char* INDEX_RHNSWSQ;
 extern const char* INDEX_ANNOY;
+extern const char* INDEX_NGTPANNG;
+extern const char* INDEX_NGTONNG;
 }  // namespace IndexEnum
 
 enum class IndexMode { MODE_CPU = 0, MODE_GPU = 1 };

core/src/index/knowhere/knowhere/index/vector_index/ConfAdapter.cpp

@@ -389,5 +389,37 @@ ANNOYConfAdapter::CheckSearch(Config& oricfg, const IndexType type, const IndexM
     return ConfAdapter::CheckSearch(oricfg, type, mode);
 }
 
+bool
+NGTPANNGConfAdapter::CheckTrain(Config& oricfg, const IndexMode mode) {
+    static std::vector<std::string> METRICS{knowhere::Metric::L2, knowhere::Metric::HAMMING, knowhere::Metric::JACCARD};
+
+    CheckIntByRange(knowhere::meta::ROWS, DEFAULT_MIN_ROWS, DEFAULT_MAX_ROWS);
+    CheckIntByRange(knowhere::meta::DIM, DEFAULT_MIN_DIM, DEFAULT_MAX_DIM);
+    CheckStrByValues(knowhere::Metric::TYPE, METRICS);
+
+    return true;
+}
+
+bool
+NGTPANNGConfAdapter::CheckSearch(Config& oricfg, const IndexType type, const IndexMode mode) {
+    return ConfAdapter::CheckSearch(oricfg, type, mode);
+}
+
+bool
+NGTONNGConfAdapter::CheckTrain(Config& oricfg, const IndexMode mode) {
+    static std::vector<std::string> METRICS{knowhere::Metric::L2, knowhere::Metric::HAMMING, knowhere::Metric::JACCARD};
+
+    CheckIntByRange(knowhere::meta::ROWS, DEFAULT_MIN_ROWS, DEFAULT_MAX_ROWS);
+    CheckIntByRange(knowhere::meta::DIM, DEFAULT_MIN_DIM, DEFAULT_MAX_DIM);
+    CheckStrByValues(knowhere::Metric::TYPE, METRICS);
+
+    return true;
+}
+
+bool
+NGTONNGConfAdapter::CheckSearch(Config& oricfg, const IndexType type, const IndexMode mode) {
+    return ConfAdapter::CheckSearch(oricfg, type, mode);
+}
+
 }  // namespace knowhere
 }  // namespace milvus

core/src/index/knowhere/knowhere/index/vector_index/ConfAdapter.h

@@ -126,5 +126,24 @@ class RHNSWSQConfAdapter : public ConfAdapter {
     bool
     CheckSearch(Config& oricfg, const IndexType type, const IndexMode mode) override;
 };
+
+class NGTPANNGConfAdapter : public ConfAdapter {
+ public:
+    bool
+    CheckTrain(Config& oricfg, const IndexMode mode) override;
+
+    bool
+    CheckSearch(Config& oricfg, const IndexType type, const IndexMode mode) override;
+};
+
+class NGTONNGConfAdapter : public ConfAdapter {
+ public:
+    bool
+    CheckTrain(Config& oricfg, const IndexMode mode) override;
+
+    bool
+    CheckSearch(Config& oricfg, const IndexType type, const IndexMode mode) override;
+};
+
 }  // namespace knowhere
 }  // namespace milvus

core/src/index/knowhere/knowhere/index/vector_index/ConfAdapterMgr.cpp

@@ -53,6 +53,8 @@ AdapterMgr::RegisterAdapter() {
     REGISTER_CONF_ADAPTER(RHNSWFlatConfAdapter, IndexEnum::INDEX_RHNSWFlat, rhnswflat_adapter);
     REGISTER_CONF_ADAPTER(RHNSWPQConfAdapter, IndexEnum::INDEX_RHNSWPQ, rhnswpq_adapter);
     REGISTER_CONF_ADAPTER(RHNSWSQConfAdapter, IndexEnum::INDEX_RHNSWSQ, rhnswsq_adapter);
+    REGISTER_CONF_ADAPTER(NGTPANNGConfAdapter, IndexEnum::INDEX_NGTPANNG, ngtpanng_adapter);
+    REGISTER_CONF_ADAPTER(NGTONNGConfAdapter, IndexEnum::INDEX_NGTONNG, ngtonng_adapter);
 }
 
 }  // namespace knowhere

core/src/index/knowhere/knowhere/index/vector_index/IndexNGT.cpp

+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#include "knowhere/index/vector_index/IndexNGT.h"
+
+#include <omp.h>
+#include <sstream>
+#include <string>
+
+#include "knowhere/index/vector_index/adapter/VectorAdapter.h"
+#include "knowhere/index/vector_index/helpers/IndexParameter.h"
+
+namespace milvus {
+namespace knowhere {
+
+BinarySet
+IndexNGT::Serialize(const Config& config) {
+    if (!index_) {
+        KNOWHERE_THROW_MSG("index not initialize or trained");
+    }
+    std::stringstream obj, grp, prf, tre;
+    index_->saveIndex(obj, grp, prf, tre);
+
+    auto obj_str = obj.str();
+    auto grp_str = grp.str();
+    auto prf_str = prf.str();
+    auto tre_str = tre.str();
+    uint64_t obj_size = obj_str.size();
+    uint64_t grp_size = grp_str.size();
+    uint64_t prf_size = prf_str.size();
+    uint64_t tre_size = tre_str.size();
+
+    std::shared_ptr<uint8_t[]> obj_data(new uint8_t[obj_size]);
+    memcpy(obj_data.get(), obj_str.data(), obj_size);
+    std::shared_ptr<uint8_t[]> grp_data(new uint8_t[grp_size]);
+    memcpy(grp_data.get(), grp_str.data(), grp_size);
+    std::shared_ptr<uint8_t[]> prf_data(new uint8_t[prf_size]);
+    memcpy(prf_data.get(), prf_str.data(), prf_size);
+    std::shared_ptr<uint8_t[]> tre_data(new uint8_t[tre_size]);
+    memcpy(tre_data.get(), tre_str.data(), tre_size);
+
+    BinarySet res_set;
+    res_set.Append("ngt_obj_data", obj_data, obj_size);
+    res_set.Append("ngt_grp_data", grp_data, grp_size);
+    res_set.Append("ngt_prf_data", prf_data, prf_size);
+    res_set.Append("ngt_tre_data", tre_data, tre_size);
+    return res_set;
+}
+
+void
+IndexNGT::Load(const BinarySet& index_binary) {
+    auto obj_data = index_binary.GetByName("ngt_obj_data");
+    std::string obj_str(reinterpret_cast<char*>(obj_data->data.get()), obj_data->size);
+
+    auto grp_data = index_binary.GetByName("ngt_grp_data");
+    std::string grp_str(reinterpret_cast<char*>(grp_data->data.get()), grp_data->size);
+
+    auto prf_data = index_binary.GetByName("ngt_prf_data");
+    std::string prf_str(reinterpret_cast<char*>(prf_data->data.get()), prf_data->size);
+
+    auto tre_data = index_binary.GetByName("ngt_tre_data");
+    std::string tre_str(reinterpret_cast<char*>(tre_data->data.get()), tre_data->size);
+
+    std::stringstream obj(obj_str);
+    std::stringstream grp(grp_str);
+    std::stringstream prf(prf_str);
+    std::stringstream tre(tre_str);
+
+    index_ = std::shared_ptr<NGT::Index>(NGT::Index::loadIndex(obj, grp, prf, tre));
+}
+
+void
+IndexNGT::BuildAll(const DatasetPtr& dataset_ptr, const Config& config) {
+    KNOWHERE_THROW_MSG("IndexNGT has no implementation of BuildAll, please use IndexNGT(PANNG/ONNG) instead!");
+}
+
+#if 0
+void
+IndexNGT::Train(const DatasetPtr& dataset_ptr, const Config& config) {
+    KNOWHERE_THROW_MSG("IndexNGT has no implementation of Train, please use IndexNGT(PANNG/ONNG) instead!");
+    GET_TENSOR_DATA_DIM(dataset_ptr);
+
+    NGT::Property prop;
+    prop.setDefaultForCreateIndex();
+    prop.dimension = dim;
+
+    MetricType metric_type = config[Metric::TYPE];
+
+    if (metric_type == Metric::L2)
+        prop.distanceType = NGT::Index::Property::DistanceType::DistanceTypeL2;
+    else if (metric_type == Metric::HAMMING)
+        prop.distanceType = NGT::Index::Property::DistanceType::DistanceTypeHamming;
+    else if (metric_type == Metric::JACCARD)
+        prop.distanceType = NGT::Index::Property::DistanceType::DistanceTypeJaccard;
+    else
+        KNOWHERE_THROW_MSG("Metric type not supported: " + metric_type);
+    index_ =
+        std::shared_ptr<NGT::Index>(NGT::Index::createGraphAndTree(reinterpret_cast<const float*>(p_data), prop, rows));
+}
+
+void
+IndexNGT::AddWithoutIds(const DatasetPtr& dataset_ptr, const Config& config) {
+    if (!index_) {
+        KNOWHERE_THROW_MSG("index not initialize");
+    }
+    GET_TENSOR_DATA(dataset_ptr);
+
+    index_->append(reinterpret_cast<const float*>(p_data), rows);
+}
+#endif
+
+DatasetPtr
+IndexNGT::Query(const DatasetPtr& dataset_ptr, const Config& config) {
+    if (!index_) {
+        KNOWHERE_THROW_MSG("index not initialize");
+    }
+    GET_TENSOR_DATA(dataset_ptr);
+
+    size_t k = config[meta::TOPK].get<int64_t>();
+    size_t id_size = sizeof(int64_t) * k;
+    size_t dist_size = sizeof(float) * k;
+    auto p_id = static_cast<int64_t*>(malloc(id_size * rows));
+    auto p_dist = static_cast<float*>(malloc(dist_size * rows));
+
+    NGT::Command::SearchParameter sp;
+    sp.size = k;
+
+    faiss::ConcurrentBitsetPtr blacklist = GetBlacklist();
+
+#pragma omp parallel for
+    for (unsigned int i = 0; i < rows; ++i) {
+        const float* single_query = reinterpret_cast<float*>(const_cast<void*>(p_data)) + i * Dim();
+
+        NGT::Object* object = index_->allocateObject(single_query, Dim());
+        NGT::SearchContainer sc(*object);
+
+        double epsilon = sp.beginOfEpsilon;
+
+        NGT::ObjectDistances res;
+        sc.setResults(&res);
+        sc.setSize(sp.size);
+        sc.setRadius(sp.radius);
+
+        if (sp.accuracy > 0.0) {
+            sc.setExpectedAccuracy(sp.accuracy);
+        } else {
+            sc.setEpsilon(epsilon);
+        }
+        sc.setEdgeSize(sp.edgeSize);
+
+        try {
+            index_->search(sc, blacklist);
+        } catch (NGT::Exception& err) {
+            KNOWHERE_THROW_MSG("Query failed");
+        }
+
+        auto local_id = p_id + i * k;
+        auto local_dist = p_dist + i * k;
+
+        int64_t res_num = res.size();
+        for (int64_t idx = 0; idx < res_num; ++idx) {
+            *(local_id + idx) = res[idx].id - 1;
+            *(local_dist + idx) = res[idx].distance;
+        }
+        while (res_num < static_cast<int64_t>(k)) {
+            *(local_id + res_num) = -1;
+            *(local_dist + res_num) = 1.0 / 0.0;
+        }
+        index_->deleteObject(object);
+    }
+
+    auto res_ds = std::make_shared<Dataset>();
+    res_ds->Set(meta::IDS, p_id);
+    res_ds->Set(meta::DISTANCE, p_dist);
+    return res_ds;
+}
+
+int64_t
+IndexNGT::Count() {
+    if (!index_) {
+        KNOWHERE_THROW_MSG("index not initialize");
+    }
+    return index_->getNumberOfVectors();
+}
+
+int64_t
+IndexNGT::Dim() {
+    if (!index_) {
+        KNOWHERE_THROW_MSG("index not initialize");
+    }
+    return index_->getDimension();
+}
+
+}  // namespace knowhere
+}  // namespace milvus

core/src/index/knowhere/knowhere/index/vector_index/IndexNGT.h

+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#pragma once
+
+#include <NGT/lib/NGT/Command.h>
+#include <NGT/lib/NGT/Common.h>
+#include <NGT/lib/NGT/Index.h>
+
+#include <knowhere/common/Exception.h>
+#include <knowhere/index/IndexType.h>
+#include <knowhere/index/vector_index/VecIndex.h>
+#include <memory>
+
+namespace milvus {
+namespace knowhere {
+
+class IndexNGT : public VecIndex {
+ public:
+    IndexNGT() {
+        index_type_ = IndexEnum::INVALID;
+    }
+
+    BinarySet
+    Serialize(const Config& config) override;
+
+    void
+    Load(const BinarySet& index_binary) override;
+
+    void
+    BuildAll(const DatasetPtr& dataset_ptr, const Config& config) override;
+
+    void
+    Train(const DatasetPtr& dataset_ptr, const Config& config) override {
+        KNOWHERE_THROW_MSG("NGT not support add item dynamically, please invoke BuildAll interface.");
+    }
+
+    void
+    Add(const DatasetPtr& dataset_ptr, const Config& config) override {
+        KNOWHERE_THROW_MSG("NGT not support add item dynamically, please invoke BuildAll interface.");
+    }
+
+    void
+    AddWithoutIds(const DatasetPtr& dataset_ptr, const Config& config) override {
+        KNOWHERE_THROW_MSG("Incremental index is not supported");
+    }
+
+    DatasetPtr
+    Query(const DatasetPtr& dataset_ptr, const Config& config) override;
+
+    int64_t
+    Count() override;
+
+    int64_t
+    Dim() override;
+
+ protected:
+    std::shared_ptr<NGT::Index> index_ = nullptr;
+};
+
+}  // namespace knowhere
+}  // namespace milvus

core/src/index/knowhere/knowhere/index/vector_index/IndexNGTONNG.cpp

+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#include "knowhere/index/vector_index/IndexNGTONNG.h"
+
+#include "NGT/lib/NGT/GraphOptimizer.h"
+
+#include "knowhere/index/vector_index/adapter/VectorAdapter.h"
+#include "knowhere/index/vector_index/helpers/IndexParameter.h"
+
+#include <memory>
+
+namespace milvus {
+namespace knowhere {
+
+void
+IndexNGTONNG::BuildAll(const DatasetPtr& dataset_ptr, const Config& config) {
+    GET_TENSOR_DATA_DIM(dataset_ptr);
+
+    NGT::Property prop;
+    prop.setDefaultForCreateIndex();
+    prop.dimension = dim;
+    prop.edgeSizeForCreation = 20;
+    prop.insertionRadiusCoefficient = 1.0;
+
+    MetricType metric_type = config[Metric::TYPE];
+
+    if (metric_type == Metric::L2) {
+        prop.distanceType = NGT::Index::Property::DistanceType::DistanceTypeL2;
+    } else if (metric_type == Metric::HAMMING) {
+        prop.distanceType = NGT::Index::Property::DistanceType::DistanceTypeHamming;
+    } else if (metric_type == Metric::JACCARD) {
+        prop.distanceType = NGT::Index::Property::DistanceType::DistanceTypeJaccard;
+    } else {
+        KNOWHERE_THROW_MSG("Metric type not supported: " + metric_type);
+    }
+    index_ =
+        std::shared_ptr<NGT::Index>(NGT::Index::createGraphAndTree(reinterpret_cast<const float*>(p_data), prop, rows));
+
+    // reconstruct graph
+    NGT::GraphOptimizer graphOptimizer(true);
+
+    size_t number_of_outgoing_edges = 5;
+    size_t number_of_incoming_edges = 30;
+    size_t number_of_queries = 1000;
+    size_t number_of_res = 20;
+
+    graphOptimizer.shortcutReduction = true;
+    graphOptimizer.searchParameterOptimization = true;
+    graphOptimizer.prefetchParameterOptimization = false;
+    graphOptimizer.accuracyTableGeneration = false;
+    graphOptimizer.margin = 0.2;
+    graphOptimizer.gtEpsilon = 0.1;
+
+    graphOptimizer.set(number_of_outgoing_edges, number_of_incoming_edges, number_of_queries, number_of_res);
+
+    graphOptimizer.execute(*index_);
+}
+
+}  // namespace knowhere
+}  // namespace milvus

core/src/index/knowhere/knowhere/index/vector_index/IndexNGTONNG.h

+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#pragma once
+
+#include "knowhere/index/vector_index/IndexNGT.h"
+
+namespace milvus {
+namespace knowhere {
+
+class IndexNGTONNG : public IndexNGT {
+ public:
+    IndexNGTONNG() {
+        index_type_ = IndexEnum::INDEX_NGTONNG;
+    }
+
+    void
+    BuildAll(const DatasetPtr& dataset_ptr, const Config& config) override;
+};
+
+}  // namespace knowhere
+}  // namespace milvus

core/src/index/knowhere/knowhere/index/vector_index/IndexNGTPANNG.cpp

+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#include "knowhere/index/vector_index/IndexNGTPANNG.h"
+#include "knowhere/index/vector_index/adapter/VectorAdapter.h"
+#include "knowhere/index/vector_index/helpers/IndexParameter.h"
+
+#include <memory>
+
+namespace milvus {
+namespace knowhere {
+
+void
+IndexNGTPANNG::BuildAll(const DatasetPtr& dataset_ptr, const Config& config) {
+    GET_TENSOR_DATA_DIM(dataset_ptr);
+
+    NGT::Property prop;
+    prop.setDefaultForCreateIndex();
+    prop.dimension = dim;
+
+    MetricType metric_type = config[Metric::TYPE];
+
+    if (metric_type == Metric::L2) {
+        prop.distanceType = NGT::Index::Property::DistanceType::DistanceTypeL2;
+    } else if (metric_type == Metric::HAMMING) {
+        prop.distanceType = NGT::Index::Property::DistanceType::DistanceTypeHamming;
+    } else if (metric_type == Metric::JACCARD) {
+        prop.distanceType = NGT::Index::Property::DistanceType::DistanceTypeJaccard;
+    } else {
+        KNOWHERE_THROW_MSG("Metric type not supported: " + metric_type);
+    }
+    index_ =
+        std::shared_ptr<NGT::Index>(NGT::Index::createGraphAndTree(reinterpret_cast<const float*>(p_data), prop, rows));
+
+    size_t force_removed_edge_size = 60;
+    size_t selective_removed_edge_size = 30;
+
+    // prune
+    auto& graph = dynamic_cast<NGT::GraphIndex&>(index_->getIndex());
+    for (size_t id = 1; id < graph.repository.size(); id++) {
+        try {
+            NGT::GraphNode& node = *graph.getNode(id);
+            if (node.size() >= force_removed_edge_size) {
+                node.resize(force_removed_edge_size);
+            }
+            if (node.size() >= selective_removed_edge_size) {
+                size_t rank = 0;
+                for (auto i = node.begin(); i != node.end(); ++rank) {
+                    if (rank >= selective_removed_edge_size) {
+                        bool found = false;
+                        for (size_t t1 = 0; t1 < node.size() && found == false; ++t1) {
+                            if (t1 >= selective_removed_edge_size) {
+                                break;
+                            }
+                            if (rank == t1) {
+                                continue;
+                            }
+                            NGT::GraphNode& node2 = *graph.getNode(node[t1].id);
+                            for (size_t t2 = 0; t2 < node2.size(); ++t2) {
+                                if (t2 >= selective_removed_edge_size) {
+                                    break;
+                                }
+                                if (node2[t2].id == (*i).id) {
+                                    found = true;
+                                    break;
+                                }
+                            }  // for
+                        }      // for
+                        if (found) {
+                            // remove
+                            i = node.erase(i);
+                            continue;
+                        }
+                    }
+                    i++;
+                }  // for
+            }
+        } catch (NGT::Exception& err) {
+            std::cerr << "Graph::search: Warning. Cannot get the node. ID=" << id << ":" << err.what() << std::endl;
+            continue;
+        }
+    }
+}
+
+}  // namespace knowhere
+}  // namespace milvus

core/src/index/knowhere/knowhere/index/vector_index/IndexNGTPANNG.h

+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#pragma once
+
+#include "knowhere/index/vector_index/IndexNGT.h"
+
+namespace milvus {
+namespace knowhere {
+
+class IndexNGTPANNG : public IndexNGT {
+ public:
+    IndexNGTPANNG() {
+        index_type_ = IndexEnum::INDEX_NGTPANNG;
+    }
+
+    void
+    BuildAll(const DatasetPtr& dataset_ptr, const Config& config) override;
+};
+
+}  // namespace knowhere
+}  // namespace milvus

core/src/index/knowhere/knowhere/index/vector_index/VecIndexFactory.cpp

@@ -21,6 +21,8 @@
 #include "knowhere/index/vector_index/IndexIVF.h"
 #include "knowhere/index/vector_index/IndexIVFPQ.h"
 #include "knowhere/index/vector_index/IndexIVFSQ.h"
+#include "knowhere/index/vector_index/IndexNGTONNG.h"
+#include "knowhere/index/vector_index/IndexNGTPANNG.h"
 #include "knowhere/index/vector_index/IndexRHNSWFlat.h"
 #include "knowhere/index/vector_index/IndexRHNSWPQ.h"
 #include "knowhere/index/vector_index/IndexRHNSWSQ.h"
@@ -99,6 +101,10 @@ VecIndexFactory::CreateVecIndex(const IndexType& type, const IndexMode mode) {
         return std::make_shared<knowhere::IndexRHNSWPQ>();
     } else if (type == IndexEnum::INDEX_RHNSWSQ) {
         return std::make_shared<knowhere::IndexRHNSWSQ>();
+    } else if (type == IndexEnum::INDEX_NGTPANNG) {
+        return std::make_shared<knowhere::IndexNGTPANNG>();
+    } else if (type == IndexEnum::INDEX_NGTONNG) {
+        return std::make_shared<knowhere::IndexNGTONNG>();
     } else {
         return nullptr;
     }

core/src/index/thirdparty/NGT/CMakeLists.txt

+if(APPLE)
+  cmake_minimum_required(VERSION 3.0)
+else()
+  cmake_minimum_required(VERSION 2.8)
+endif()
+
+project(ngt)
+
+file(STRINGS "VERSION" ngt_VERSION)
+message(STATUS "VERSION: ${ngt_VERSION}")
+string(REGEX MATCH "^[0-9]+" ngt_VERSION_MAJOR ${ngt_VERSION})
+
+set(ngt_VERSION ${ngt_VERSION})
+set(ngt_SOVERSION ${ngt_VERSION_MAJOR})
+
+if (NOT CMAKE_BUILD_TYPE)
+	set (CMAKE_BUILD_TYPE "Release")
+endif (NOT CMAKE_BUILD_TYPE)
+string(TOLOWER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_LOWER)
+message(STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}")
+message(STATUS "CMAKE_BUILD_TYPE_LOWER: ${CMAKE_BUILD_TYPE_LOWER}")
+
+if(${UNIX})
+    set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
+
+    if(CMAKE_VERSION VERSION_LESS 3.1)
+        set(BASE_OPTIONS "-Wall -std=gnu++0x -lrt")
+
+        if(${NGT_AVX_DISABLED})
+            message(STATUS "AVX will not be used to compute distances.")
+        endif()
+
+        if(${NGT_OPENMP_DISABLED})
+            message(STATUS "OpenMP is disabled.")
+        else()
+            set(BASE_OPTIONS "${BASE_OPTIONS} -fopenmp")
+        endif()
+
+        set(CMAKE_CXX_FLAGS_DEBUG "-g ${BASE_OPTIONS}")
+
+        if(${NGT_MARCH_NATIVE_DISABLED})
+            message(STATUS "Compile option -march=native is disabled.")
+            set(CMAKE_CXX_FLAGS_RELEASE "-O2 ${BASE_OPTIONS}")
+        else()
+            set(CMAKE_CXX_FLAGS_RELEASE "-O3 -march=native ${BASE_OPTIONS}")
+        endif()
+    else()
+        if (CMAKE_BUILD_TYPE_LOWER STREQUAL "release")
+            set(CMAKE_CXX_FLAGS_RELEASE "")
+            if(${NGT_MARCH_NATIVE_DISABLED})
+                message(STATUS "Compile option -march=native is disabled.")
+                add_compile_options(-O2 -DNDEBUG)
+            else()
+                add_compile_options(-Ofast -march=native -DNDEBUG)
+            endif()
+        endif()
+	add_compile_options(-Wall)
+        if(${NGT_AVX_DISABLED})
+            message(STATUS "AVX will not be used to compute distances.")
+        endif()
+        if(${NGT_OPENMP_DISABLED})
+            message(STATUS "OpenMP is disabled.")
+        else()
+            if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
+                if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "8.1.0")
+                    message(FATAL_ERROR "Insufficient AppleClang version")
+                endif()
+                cmake_minimum_required(VERSION 3.16)
+            endif()
+            find_package(OpenMP REQUIRED)
+            set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+        endif()
+        set(CMAKE_CXX_STANDARD 11) # for std::unordered_set, std::unique_ptr
+        set(CMAKE_CXX_STANDARD_REQUIRED ON)
+	find_package(Threads REQUIRED)
+    endif() 
+
+    add_subdirectory("${PROJECT_SOURCE_DIR}/lib")
+endif( ${UNIX} )

core/src/index/thirdparty/NGT/LICENSE

+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

core/src/index/thirdparty/NGT/VERSION

+1.12.0

core/src/index/thirdparty/NGT/lib/CMakeLists.txt

+if( ${UNIX} )
+	add_subdirectory(${PROJECT_SOURCE_DIR}/lib/NGT)
+endif()

core/src/index/thirdparty/NGT/lib/NGT/ArrayFile.cpp

+//
+// Copyright (C) 2015-2020 Yahoo Japan Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include "ArrayFile.h"
+#include <iostream>
+#include <assert.h>
+
+class ItemID {
+public:
+  void serialize(std::ostream &os, NGT::ObjectSpace *ospace = 0) {
+    os.write((char*)&value, sizeof(value));
+  }
+  void deserialize(std::istream &is, NGT::ObjectSpace *ospace = 0) {
+    is.read((char*)&value, sizeof(value));
+  }
+  static size_t getSerializedDataSize() {
+    return sizeof(uint64_t);
+  }
+  uint64_t value;
+};
+
+void
+sampleForUsage() {
+  {
+    ArrayFile<ItemID> itemIDFile;
+    itemIDFile.create("test.data", ItemID::getSerializedDataSize());
+    itemIDFile.open("test.data");
+    ItemID itemID;
+    size_t id;
+
+    id = 1;
+    itemID.value = 4910002490100;
+    itemIDFile.put(id, itemID);
+    itemID.value = 0;
+    itemIDFile.get(id, itemID);
+    std::cerr << "value=" << itemID.value << std::endl;
+    assert(itemID.value == 4910002490100);
+
+    id = 2;
+    itemID.value = 4910002490101;
+    itemIDFile.put(id, itemID);
+    itemID.value = 0;
+    itemIDFile.get(id, itemID);
+    std::cerr << "value=" << itemID.value << std::endl;
+    assert(itemID.value == 4910002490101);
+
+    itemID.value = 4910002490102;
+    id = itemIDFile.insert(itemID);
+    itemID.value = 0;
+    itemIDFile.get(id, itemID);
+    std::cerr << "value=" << itemID.value << std::endl;
+    assert(itemID.value == 4910002490102);
+
+    itemIDFile.close();
+  }
+  {
+    ArrayFile<ItemID> itemIDFile;
+    itemIDFile.create("test.data", ItemID::getSerializedDataSize());
+    itemIDFile.open("test.data");
+    ItemID itemID;
+    size_t id;
+
+    id = 10;
+    itemIDFile.get(id, itemID);
+    std::cerr << "value=" << itemID.value << std::endl;
+    assert(itemID.value == 4910002490100);
+
+    id = 20;
+    itemIDFile.get(id, itemID);
+    std::cerr << "value=" << itemID.value << std::endl;
+    assert(itemID.value == 4910002490101);
+  }
+
+}
+
+

core/src/index/thirdparty/NGT/lib/NGT/ArrayFile.h

+//
+// Copyright (C) 2015-2020 Yahoo Japan Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#pragma once
+
+#include <fstream>
+#include <string>
+#include <cstddef>
+#include <stdint.h>
+#include <iostream>
+#include <stdexcept>
+#include <cerrno>
+#include <cstring>
+
+namespace NGT {
+  class ObjectSpace;
+};
+
+template <class TYPE>
+class ArrayFile {
+ private:
+  struct FileHeadStruct {
+    size_t recordSize;
+    uint64_t extraData; // reserve
+  };
+
+  struct RecordStruct {
+    bool deleteFlag;
+    uint64_t extraData; // reserve    
+  };
+
+  bool _isOpen;  
+  std::fstream _stream;
+  FileHeadStruct _fileHead;
+
+  bool _readFileHead();
+  pthread_mutex_t _mutex;
+  
+ public:
+  ArrayFile();
+  ~ArrayFile();
+  bool create(const std::string &file, size_t recordSize);
+  bool open(const std::string &file);
+  void close();
+  size_t insert(TYPE &data, NGT::ObjectSpace *objectSpace = 0);
+  void put(const size_t id, TYPE &data, NGT::ObjectSpace *objectSpace = 0);
+  bool get(const size_t id, TYPE &data, NGT::ObjectSpace *objectSpace = 0);
+  void remove(const size_t id);
+  bool isOpen() const;
+  size_t size();
+  size_t getRecordSize() { return _fileHead.recordSize; }
+};
+
+
+// constructor 
+template <class TYPE>
+ArrayFile<TYPE>::ArrayFile()
+  : _isOpen(false), _mutex((pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER){
+    if(pthread_mutex_init(&_mutex, NULL) < 0) throw std::runtime_error("pthread init error.");
+}
+
+// destructor
+template <class TYPE>
+ArrayFile<TYPE>::~ArrayFile() {
+  pthread_mutex_destroy(&_mutex);
+  close();
+}
+
+template <class TYPE>
+bool ArrayFile<TYPE>::create(const std::string &file, size_t recordSize) {
+  std::fstream tmpstream;
+  tmpstream.open(file.c_str());  
+  if(tmpstream){
+    return false;
+  }
+  
+  tmpstream.open(file.c_str(), std::ios::out);
+  tmpstream.seekp(0, std::ios::beg);
+  FileHeadStruct fileHead = {recordSize, 0};
+  tmpstream.write((char *)(&fileHead), sizeof(FileHeadStruct));
+  tmpstream.close();
+  
+  return true;
+}
+
+template <class TYPE>
+bool ArrayFile<TYPE>::open(const std::string &file) {
+  _stream.open(file.c_str(), std::ios::in | std::ios::out);  
+  if(!_stream){
+    _isOpen = false;    
+    return false;
+  }
+  _isOpen = true;
+
+  bool ret = _readFileHead();
+  return ret;
+}
+
+template <class TYPE>
+void ArrayFile<TYPE>::close(){
+  _stream.close();
+  _isOpen = false;  
+}
+
+template <class TYPE>
+size_t ArrayFile<TYPE>::insert(TYPE &data, NGT::ObjectSpace *objectSpace) {
+  _stream.seekp(sizeof(RecordStruct), std::ios::end);
+  int64_t write_pos = _stream.tellg();
+  for(size_t i = 0; i < _fileHead.recordSize; i++) { _stream.write("", 1); }
+  _stream.seekp(write_pos, std::ios::beg);
+  data.serialize(_stream, objectSpace);
+  
+  int64_t offset_pos = _stream.tellg();
+  offset_pos -= sizeof(FileHeadStruct);
+  size_t id = offset_pos / (sizeof(RecordStruct) + _fileHead.recordSize);
+  if(offset_pos % (sizeof(RecordStruct) + _fileHead.recordSize) == 0){
+    id -= 1;
+  }
+  
+  return id;
+}
+
+template <class TYPE>
+void ArrayFile<TYPE>::put(const size_t id, TYPE &data, NGT::ObjectSpace *objectSpace) {
+  uint64_t offset_pos = (id * (sizeof(RecordStruct) + _fileHead.recordSize)) + sizeof(FileHeadStruct);
+  offset_pos += sizeof(RecordStruct);
+  _stream.seekp(offset_pos, std::ios::beg);
+  
+  for(size_t i = 0; i < _fileHead.recordSize; i++) { _stream.write("", 1); }
+  _stream.seekp(offset_pos, std::ios::beg); 
+  data.serialize(_stream, objectSpace);
+}
+
+template <class TYPE>
+bool ArrayFile<TYPE>::get(const size_t id, TYPE &data, NGT::ObjectSpace *objectSpace) {
+  pthread_mutex_lock(&_mutex);
+
+  if( size() <= id ){
+    pthread_mutex_unlock(&_mutex);    
+    return false;
+  }
+  
+  uint64_t offset_pos = (id * (sizeof(RecordStruct) + _fileHead.recordSize)) + sizeof(FileHeadStruct);
+  offset_pos += sizeof(RecordStruct);  
+  _stream.seekg(offset_pos, std::ios::beg);
+  if (!_stream.fail()) {
+    data.deserialize(_stream, objectSpace);
+  }
+  if (_stream.fail()) {
+    const int trialCount = 10;
+    for (int tc = 0; tc < trialCount; tc++) {
+      _stream.clear();
+      _stream.seekg(offset_pos, std::ios::beg);
+      if (_stream.fail()) {
+	continue;
+      }
+      data.deserialize(_stream, objectSpace);
+      if (_stream.fail()) {
+	continue;
+      } else {
+	break;
+      }
+    }
+    if (_stream.fail()) {
+      throw std::runtime_error("ArrayFile::get: Error!");
+    }
+  }
+
+  pthread_mutex_unlock(&_mutex);
+  return true;
+}
+
+template <class TYPE>
+void ArrayFile<TYPE>::remove(const size_t id) {
+  uint64_t offset_pos = (id * (sizeof(RecordStruct) + _fileHead.recordSize)) + sizeof(FileHeadStruct);  
+  _stream.seekp(offset_pos, std::ios::beg);
+  RecordStruct recordHead = {1, 0};
+  _stream.write((char *)(&recordHead), sizeof(RecordStruct));
+}
+
+template <class TYPE>
+bool ArrayFile<TYPE>::isOpen() const
+{
+  return _isOpen;
+}
+
+template <class TYPE>
+size_t ArrayFile<TYPE>::size()
+{
+  _stream.seekp(0, std::ios::end);
+  int64_t offset_pos = _stream.tellg();
+  offset_pos -= sizeof(FileHeadStruct);
+  size_t num = offset_pos / (sizeof(RecordStruct) + _fileHead.recordSize);
+
+  return num; 
+}
+
+template <class TYPE>
+bool ArrayFile<TYPE>::_readFileHead() {
+  _stream.seekp(0, std::ios::beg);
+  _stream.read((char *)(&_fileHead), sizeof(FileHeadStruct));
+  if(_stream.bad()){
+    return false;
+  }
+  return true;
+}
+

core/src/index/thirdparty/NGT/lib/NGT/CMakeLists.txt

+if( ${UNIX} )
+	option(NGT_SHARED_MEMORY_ALLOCATOR "enable shared memory" OFF)
+	configure_file(${CMAKE_CURRENT_SOURCE_DIR}/defines.h.in ${CMAKE_CURRENT_BINARY_DIR}/defines.h)
+	include_directories("${CMAKE_CURRENT_BINARY_DIR}" "${PROJECT_SOURCE_DIR}/lib" "${PROJECT_BINARY_DIR}/lib/")
+	include_directories("${PROJECT_SOURCE_DIR}/../")
+
+	file(GLOB NGT_SOURCES *.cpp)
+	file(GLOB HEADER_FILES *.h *.hpp)
+	file(GLOB NGTQ_HEADER_FILES NGTQ/*.h NGTQ/*.hpp)
+
+	add_library(ngtstatic STATIC ${NGT_SOURCES})
+	set_target_properties(ngtstatic PROPERTIES OUTPUT_NAME ngt) 
+	set_target_properties(ngtstatic PROPERTIES COMPILE_FLAGS "-fPIC")
+	target_link_libraries(ngtstatic)
+	if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
+		target_link_libraries(ngtstatic OpenMP::OpenMP_CXX)
+	endif()
+
+	add_library(ngt SHARED ${NGT_SOURCES})
+	set_target_properties(ngt PROPERTIES VERSION ${ngt_VERSION}) 
+	set_target_properties(ngt PROPERTIES SOVERSION ${ngt_SOVERSION}) 
+	add_dependencies(ngt ngtstatic) 
+	if(${APPLE})
+		if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
+			target_link_libraries(ngt OpenMP::OpenMP_CXX)
+		else()
+			target_link_libraries(ngt gomp)
+		endif()
+	else(${APPLE})
+		target_link_libraries(ngt gomp rt)
+	endif(${APPLE})
+
+	install(TARGETS
+		ngt
+		ngtstatic
+		RUNTIME DESTINATION bin
+		LIBRARY DESTINATION lib
+		ARCHIVE DESTINATION lib)
+
+endif()

core/src/index/thirdparty/NGT/lib/NGT/Capi.h

+//
+// Copyright (C) 2015-2020 Yahoo Japan Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#pragma once
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+#include <stdint.h>
+#include <stdbool.h>
+
+typedef unsigned int ObjectID;
+typedef void* NGTIndex;
+typedef void* NGTProperty;
+typedef void* NGTObjectSpace;
+typedef void* NGTObjectDistances;
+typedef void* NGTError;
+typedef void* NGTOptimizer;
+
+typedef struct {
+  ObjectID id;
+  float distance;
+} NGTObjectDistance;
+
+typedef struct {
+  float		*query;
+  size_t	size;		// # of returned objects
+  float		epsilon;
+  float		accuracy;	// expected accuracy
+  float		radius;
+  size_t	edge_size;	// # of edges to explore for each node
+} NGTQuery;
+
+typedef struct {
+  size_t	no_of_queries;
+  size_t	no_of_results;
+  size_t	no_of_threads;
+  float		target_accuracy;
+  size_t	target_no_of_objects;
+  size_t	no_of_sample_objects;
+  size_t	max_of_no_of_edges;
+  bool		log;
+} NGTAnngEdgeOptimizationParameter;
+
+NGTIndex ngt_open_index(const char *, NGTError);
+
+NGTIndex ngt_create_graph_and_tree(const char *, NGTProperty, NGTError);
+
+NGTIndex ngt_create_graph_and_tree_in_memory(NGTProperty, NGTError);
+
+NGTProperty ngt_create_property(NGTError);
+
+bool ngt_save_index(const NGTIndex, const char *, NGTError);
+
+bool ngt_get_property(const NGTIndex, NGTProperty, NGTError);
+
+int32_t ngt_get_property_dimension(NGTProperty, NGTError);
+
+bool ngt_set_property_dimension(NGTProperty, int32_t, NGTError);
+
+bool ngt_set_property_edge_size_for_creation(NGTProperty, int16_t, NGTError);
+
+bool ngt_set_property_edge_size_for_search(NGTProperty, int16_t, NGTError);
+
+int32_t ngt_get_property_object_type(NGTProperty, NGTError);
+
+bool ngt_is_property_object_type_float(int32_t);
+
+bool ngt_is_property_object_type_integer(int32_t);
+
+bool ngt_set_property_object_type_float(NGTProperty, NGTError);
+
+bool ngt_set_property_object_type_integer(NGTProperty, NGTError);
+
+bool ngt_set_property_distance_type_l1(NGTProperty, NGTError);
+
+bool ngt_set_property_distance_type_l2(NGTProperty, NGTError);
+
+bool ngt_set_property_distance_type_angle(NGTProperty, NGTError);
+
+bool ngt_set_property_distance_type_hamming(NGTProperty, NGTError);
+
+bool ngt_set_property_distance_type_jaccard(NGTProperty, NGTError);
+
+bool ngt_set_property_distance_type_cosine(NGTProperty, NGTError);
+
+bool ngt_set_property_distance_type_normalized_angle(NGTProperty, NGTError);
+
+bool ngt_set_property_distance_type_normalized_cosine(NGTProperty, NGTError);  
+  
+NGTObjectDistances ngt_create_empty_results(NGTError);
+
+bool ngt_search_index(NGTIndex, double*, int32_t, size_t, float, float, NGTObjectDistances, NGTError);
+
+bool ngt_search_index_as_float(NGTIndex, float*, int32_t, size_t, float, float, NGTObjectDistances, NGTError);
+
+bool ngt_search_index_with_query(NGTIndex, NGTQuery, NGTObjectDistances, NGTError);
+
+int32_t ngt_get_size(NGTObjectDistances, NGTError); // deprecated
+  
+uint32_t ngt_get_result_size(NGTObjectDistances, NGTError); 
+
+NGTObjectDistance ngt_get_result(const NGTObjectDistances, const uint32_t, NGTError);
+
+ObjectID ngt_insert_index(NGTIndex, double*, uint32_t, NGTError);
+
+ObjectID ngt_append_index(NGTIndex, double*, uint32_t, NGTError);
+
+ObjectID ngt_insert_index_as_float(NGTIndex, float*, uint32_t, NGTError);
+
+ObjectID ngt_append_index_as_float(NGTIndex, float*, uint32_t, NGTError);
+
+bool ngt_batch_append_index(NGTIndex, float*, uint32_t, NGTError);
+
+bool ngt_batch_insert_index(NGTIndex, float*, uint32_t, uint32_t *, NGTError);
+
+bool ngt_create_index(NGTIndex, uint32_t, NGTError);
+
+bool ngt_remove_index(NGTIndex, ObjectID, NGTError);
+
+NGTObjectSpace ngt_get_object_space(NGTIndex, NGTError);
+
+float* ngt_get_object_as_float(NGTObjectSpace, ObjectID, NGTError);
+
+uint8_t* ngt_get_object_as_integer(NGTObjectSpace, ObjectID, NGTError);
+
+void ngt_destroy_results(NGTObjectDistances);
+
+void ngt_destroy_property(NGTProperty);
+
+void ngt_close_index(NGTIndex);
+
+int16_t ngt_get_property_edge_size_for_creation(NGTProperty, NGTError);
+
+int16_t ngt_get_property_edge_size_for_search(NGTProperty, NGTError);
+
+int32_t ngt_get_property_distance_type(NGTProperty, NGTError);
+
+NGTError ngt_create_error_object();
+
+const char *ngt_get_error_string(const NGTError);
+
+void ngt_clear_error_string(NGTError);
+  
+void ngt_destroy_error_object(NGTError);
+
+NGTOptimizer ngt_create_optimizer(bool logDisabled, NGTError);
+
+bool ngt_optimizer_adjust_search_coefficients(NGTOptimizer, const char *, NGTError);
+
+bool ngt_optimizer_execute(NGTOptimizer, const char *, const char *, NGTError);
+
+bool ngt_optimizer_set(NGTOptimizer optimizer, int outgoing, int incoming, int nofqs, 
+		       float baseAccuracyFrom, float baseAccuracyTo,
+		       float rateAccuracyFrom, float rateAccuracyTo,
+		       double gte, double m, NGTError error);
+
+bool ngt_optimizer_set_minimum(NGTOptimizer optimizer, int outgoing, int incoming, 
+			       int nofqs, int nofrs, NGTError error);
+
+bool ngt_optimizer_set_extension(NGTOptimizer optimizer,
+				 float baseAccuracyFrom, float baseAccuracyTo,
+				 float rateAccuracyFrom, float rateAccuracyTo,
+				 double gte, double m, NGTError error);
+
+bool ngt_optimizer_set_processing_modes(NGTOptimizer optimizer, bool searchParameter, 
+					bool prefetchParameter, bool accuracyTable, NGTError error);
+
+void ngt_destroy_optimizer(NGTOptimizer);
+
+// refine: the specified index by searching each node.
+// epsilon, exepectedAccuracy and edgeSize: the same as the prameters for search. but if edgeSize is INT_MIN, default is used.
+// noOfEdges: if this is not 0, kNNG with k = noOfEdges is build 
+// batchSize: batch size for parallelism.
+bool ngt_refine_anng(NGTIndex index, float epsilon, float expectedAccuracy, 
+		     int noOfEdges, int edgeSize, size_t batchSize, NGTError error);
+
+// get edges of the node that is specified with id.
+bool ngt_get_edges(NGTIndex index, ObjectID id, NGTObjectDistances edges, NGTError error);
+
+// get the size of the specified object repository.
+// Since the size includes empty objects, the size is not the number of objects.
+// The size is mostly the largest ID of the objects - 1;
+uint32_t ngt_get_object_repository_size(NGTIndex index, NGTError error);
+
+// return parameters for ngt_optimize_number_of_edges. You can customize them before calling ngt_optimize_number_of_edges.
+NGTAnngEdgeOptimizationParameter ngt_get_anng_edge_optimization_parameter();
+
+// optimize the number of initial edges for ANNG that is specified with indexPath.
+// The parameter should be a struct which is returned by nt_get_optimization_parameter.
+bool ngt_optimize_number_of_edges(const char *indexPath, NGTAnngEdgeOptimizationParameter parameter, NGTError error);
+
+#ifdef __cplusplus
+}
+#endif

core/src/index/thirdparty/NGT/lib/NGT/Command.h

+//
+// Copyright (C) 2015-2020 Yahoo Japan Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+#pragma once
+
+#include	"NGT/Index.h"
+
+namespace NGT {
+
+
+class Command {
+public:
+  class SearchParameter {
+  public:
+    SearchParameter() {
+      openMode   = 'r';
+      query      = "";
+      querySize  = 0;
+      indexType  = 't';
+      size       = 20;
+      edgeSize   = -1;
+      outputMode = "-";
+      radius     = FLT_MAX;
+      step	 = 0;
+      trial      = 1;
+      beginOfEpsilon = endOfEpsilon = stepOfEpsilon = 0.1;
+      accuracy	 = 0.0;
+    }
+    SearchParameter(Args &args) { parse(args); }
+    void parse(Args &args) {
+      openMode = args.getChar("m", 'r');
+      try {
+	query = args.get("#2");
+      } catch (...) {
+	NGTThrowException("ngt: Error: Query is not specified");
+      }
+      querySize = args.getl("Q", 0);
+      indexType	= args.getChar("i", 't');
+      size	= args.getl("n", 20);
+      // edgeSize
+      // -1(default) : using the size which was specified at the index creation.
+      //  0 : no limitation for the edge size.
+      // -2('e') : automatically set it according to epsilon.
+      if (args.getChar("E", '-') == 'e') {
+	edgeSize	= -2;
+      } else {
+	edgeSize	= args.getl("E", -1);
+      }
+      outputMode	= args.getString("o", "-");
+      radius		= args.getf("r", FLT_MAX);
+      trial		= args.getl("t", 1);
+      {
+	beginOfEpsilon = endOfEpsilon = stepOfEpsilon = 0.1;
+	std::string epsilon = args.getString("e", "0.1");
+	std::vector<std::string> tokens;
+	NGT::Common::tokenize(epsilon, tokens, ":");
+	if (tokens.size() >= 1) { beginOfEpsilon = endOfEpsilon = NGT::Common::strtod(tokens[0]); }
+	if (tokens.size() >= 2) { endOfEpsilon = NGT::Common::strtod(tokens[1]); }
+	if (tokens.size() >= 3) { stepOfEpsilon = NGT::Common::strtod(tokens[2]); }
+	step = 0;
+	if (tokens.size() >= 4) { step = NGT::Common::strtol(tokens[3]); }
+      }
+      accuracy		= args.getf("a", 0.0);
+    }
+    char	openMode;
+    std::string	query;
+    size_t	querySize;
+    char	indexType;
+    int		size;
+    long	edgeSize;
+    std::string	outputMode;
+    float	radius;
+    float	beginOfEpsilon;
+    float	endOfEpsilon;
+    float	stepOfEpsilon;
+    float	accuracy;
+    size_t	step;
+    size_t	trial;
+  };
+
+  Command():debugLevel(0) {}
+
+  void create(Args &args);
+  void append(Args &args);
+  static void search(NGT::Index &index, SearchParameter &searchParameter, std::ostream &stream)
+  {
+    std::ifstream		is(searchParameter.query);
+    if (!is) {
+      std::cerr << "Cannot open the specified file. " << searchParameter.query << std::endl;
+      return;
+    }
+    search(index, searchParameter, is, stream);
+  }
+  static void search(NGT::Index &index, SearchParameter &searchParameter, std::istream &is, std::ostream &stream);
+  void search(Args &args);
+  void remove(Args &args);
+  void exportIndex(Args &args);
+  void importIndex(Args &args);
+  void prune(Args &args);
+  void reconstructGraph(Args &args);
+  void optimizeSearchParameters(Args &args);
+  void optimizeNumberOfEdgesForANNG(Args &args);
+  void refineANNG(Args &args);
+  void repair(Args &args);
+
+  void info(Args &args);
+  void setDebugLevel(int level) { debugLevel = level; }
+  int getDebugLevel() { return debugLevel; }
+
+protected:
+  int debugLevel;
+
+};
+
+}; // NGT

core/src/index/thirdparty/NGT/lib/NGT/ConcurrentBitset.cpp

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "faiss/utils/ConcurrentBitset.h"
+#include <cstring>
+
+namespace faiss {
+
+ConcurrentBitset::ConcurrentBitset(id_type_t capacity, uint8_t init_value) : capacity_(capacity), bitset_(((capacity + 8 - 1) >> 3)) {
+    if (init_value) {
+        memset(mutable_data(), init_value, (capacity + 8 - 1) >> 3);
+    }
+}
+
+std::vector<std::atomic<uint8_t>>&
+ConcurrentBitset::bitset() {
+    return bitset_;
+}
+
+ConcurrentBitset&
+ConcurrentBitset::operator&=(ConcurrentBitset& bitset) {
+    //    for (id_type_t i = 0; i < ((capacity_ + 8 -1) >> 3); ++i) {
+    //        bitset_[i].fetch_and(bitset.bitset()[i].load());
+    //    }
+
+    auto u8_1 = const_cast<uint8_t*>(data());
+    auto u8_2 = const_cast<uint8_t*>(bitset.data());
+    auto u64_1 = reinterpret_cast<uint64_t*>(u8_1);
+    auto u64_2 = reinterpret_cast<uint64_t*>(u8_2);
+
+    size_t n8 = bitset_.size();
+    size_t n64 = n8 / 8;
+
+    for (size_t i = 0; i < n64; i++) {
+        u64_1[i] &= u64_2[i];
+    }
+
+    size_t remain = n8 % 8;
+    u8_1 += n64 * 8;
+    u8_2 += n64 * 8;
+    for (size_t i = 0; i < remain; i++) {
+        u8_1[i] &= u8_2[i];
+    }
+
+    return *this;
+}
+
+std::shared_ptr<ConcurrentBitset>
+ConcurrentBitset::operator&(const std::shared_ptr<ConcurrentBitset>& bitset) {
+    auto result_bitset = std::make_shared<ConcurrentBitset>(bitset->capacity());
+
+    auto result_8 = const_cast<uint8_t*>(result_bitset->data());
+    auto result_64 = reinterpret_cast<uint64_t*>(result_8);
+
+    auto u8_1 = const_cast<uint8_t*>(data());
+    auto u8_2 = const_cast<uint8_t*>(bitset->data());
+    auto u64_1 = reinterpret_cast<uint64_t*>(u8_1);
+    auto u64_2 = reinterpret_cast<uint64_t*>(u8_2);
+
+    size_t n8 = bitset_.size();
+    size_t n64 = n8 / 8;
+
+    for (size_t i = 0; i < n64; i++) {
+        result_64[i] = u64_1[i] & u64_2[i];
+    }
+
+    size_t remain = n8 % 8;
+    u8_1 += n64 * 8;
+    u8_2 += n64 * 8;
+    result_8 += n64 * 8;
+    for (size_t i = 0; i < remain; i++) {
+         result_8[i] = u8_1[i] & u8_2[i];
+    }
+
+
+    return result_bitset;
+}
+
+ConcurrentBitset&
+ConcurrentBitset::operator|=(ConcurrentBitset& bitset) {
+    //    for (id_type_t i = 0; i < ((capacity_ + 8 -1) >> 3); ++i) {
+    //        bitset_[i].fetch_or(bitset.bitset()[i].load());
+    //    }
+
+    auto u8_1 = const_cast<uint8_t*>(data());
+    auto u8_2 = const_cast<uint8_t*>(bitset.data());
+    auto u64_1 = reinterpret_cast<uint64_t*>(u8_1);
+    auto u64_2 = reinterpret_cast<uint64_t*>(u8_2);
+
+    size_t n8 = bitset_.size();
+    size_t n64 = n8 / 8;
+
+    for (size_t i = 0; i < n64; i++) {
+        u64_1[i] |= u64_2[i];
+    }
+
+    size_t remain = n8 % 8;
+    u8_1 += n64 * 8;
+    u8_2 += n64 * 8;
+    for (size_t i = 0; i < remain; i++) {
+        u8_1[i] |= u8_2[i];
+    }
+
+    return *this;
+}
+
+std::shared_ptr<ConcurrentBitset>
+ConcurrentBitset::operator|(const std::shared_ptr<ConcurrentBitset>& bitset) {
+    auto result_bitset = std::make_shared<ConcurrentBitset>(bitset->capacity());
+
+    auto result_8 = const_cast<uint8_t*>(result_bitset->data());
+    auto result_64 = reinterpret_cast<uint64_t*>(result_8);
+
+    auto u8_1 = const_cast<uint8_t*>(data());
+    auto u8_2 = const_cast<uint8_t*>(bitset->data());
+    auto u64_1 = reinterpret_cast<uint64_t*>(u8_1);
+    auto u64_2 = reinterpret_cast<uint64_t*>(u8_2);
+
+    size_t n8 = bitset_.size();
+    size_t n64 = n8 / 8;
+
+    for (size_t i = 0; i < n64; i++) {
+        result_64[i] = u64_1[i] | u64_2[i];
+    }
+
+    size_t remain = n8 % 8;
+    u8_1 += n64 * 8;
+    u8_2 += n64 * 8;
+    result_8 += n64 * 8;
+    for (size_t i = 0; i < remain; i++) {
+        result_8[i] = u8_1[i] | u8_2[i];
+    }
+
+    return result_bitset;
+}
+
+ConcurrentBitset&
+ConcurrentBitset::operator^=(ConcurrentBitset& bitset) {
+    //    for (id_type_t i = 0; i < ((capacity_ + 8 -1) >> 3); ++i) { 
+    //        bitset_[i].fetch_xor(bitset.bitset()[i].load());
+    //    }
+
+    auto u8_1 = const_cast<uint8_t*>(data());
+    auto u8_2 = const_cast<uint8_t*>(bitset.data());
+    auto u64_1 = reinterpret_cast<uint64_t*>(u8_1);
+    auto u64_2 = reinterpret_cast<uint64_t*>(u8_2);
+
+    size_t n8 = bitset_.size();
+    size_t n64 = n8 / 8;
+
+    for (size_t i = 0; i < n64; i++) {
+        u64_1[i] &= u64_2[i];
+    }
+
+    size_t remain = n8 % 8;
+    u8_1 += n64 * 8;
+    u8_2 += n64 * 8;
+    for (size_t i = 0; i < remain; i++) {
+        u8_1[i] ^= u8_2[i];
+    }
+
+    return *this;
+}
+
+bool
+ConcurrentBitset::test(id_type_t id) {
+    return bitset_[id >> 3].load() & (0x1 << (id & 0x7));
+}
+
+void
+ConcurrentBitset::set(id_type_t id) {
+    bitset_[id >> 3].fetch_or(0x1 << (id & 0x7));
+}
+
+void
+ConcurrentBitset::clear(id_type_t id) {
+    bitset_[id >> 3].fetch_and(~(0x1 << (id & 0x7)));
+}
+
+size_t
+ConcurrentBitset::capacity() {
+    return capacity_;
+}
+
+size_t
+ConcurrentBitset::size() {
+    return ((capacity_ + 8 - 1) >> 3);
+}
+
+const uint8_t*
+ConcurrentBitset::data() {
+    return reinterpret_cast<const uint8_t*>(bitset_.data());
+}
+
+uint8_t*
+ConcurrentBitset::mutable_data() {
+    return reinterpret_cast<uint8_t*>(bitset_.data());
+}
+}  // namespace faiss

core/src/index/thirdparty/NGT/lib/NGT/GetCoreNumber.cpp

+#include "NGT/GetCoreNumber.h"
+
+namespace NGT
+{
+int getCoreNumber()
+{
+#ifndef __linux__
+    SYSTEM_INFO sys_info;
+    GetSystemInfo(&sys_info);
+    return sysInfo.dwNumberOfProcessors;
+#else
+    return get_nprocs();
+#endif
+}
+}

core/src/index/thirdparty/NGT/lib/NGT/GetCoreNumber.h

+#ifndef __linux__
+#    include "windows.h"
+#else
+
+#    include "sys/sysinfo.h"
+#    include "unistd.h"
+#endif
+
+namespace NGT
+{
+int getCoreNumber();
+}

core/src/index/thirdparty/NGT/lib/NGT/HashBasedBooleanSet.h

+//
+// Copyright (C) 2015-2020 Yahoo Japan Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#pragma once
+
+#include <iostream>
+#include <cstring>
+#include <stdint.h>
+#include <climits>
+#include <unordered_set>
+
+class HashBasedBooleanSet{
+ private:
+  uint32_t *_table;
+  uint32_t _tableSize;
+  uint32_t _mask;
+  
+  std::unordered_set<uint32_t> _stlHash;
+  
+  
+  inline uint32_t _hash1(const uint32_t value){
+    return value & _mask;
+  }
+  
+ public:
+ HashBasedBooleanSet():_table(NULL), _tableSize(0), _mask(0) {}
+
+ HashBasedBooleanSet(const uint64_t size):_table(NULL), _tableSize(0), _mask(0) {
+   size_t bitSize = 0;
+   size_t bit = size;
+   while (bit != 0) {
+     bitSize++;
+     bit >>= 1;
+   }
+   size_t bucketSize = 0x1 << ((bitSize + 4) / 2 + 3);
+   initialize(bucketSize);
+ }
+ void initialize(const uint32_t tableSize) {
+    _tableSize = tableSize;
+    _mask = _tableSize - 1;
+    const uint32_t checkValue = _hash1(tableSize);
+    if(checkValue != 0){
+      std::cerr << "[WARN] table size is not 2^N :  " <<  tableSize << std::endl;
+    }
+    
+    _table = new uint32_t[tableSize];
+    memset(_table, 0, tableSize * sizeof(uint32_t));
+  }
+  
+  ~HashBasedBooleanSet(){
+    delete[] _table;
+    _stlHash.clear();
+  }
+  
+  inline bool operator[](const uint32_t num){
+    const uint32_t hashValue = _hash1(num);
+    
+    auto v = _table[hashValue];
+    if (v == num){
+      return true;
+    }
+    if (v == 0){
+      return false;
+    }
+    if (_stlHash.count(num) <= 0) {    
+      return false;
+    }
+    return true;
+  }
+  
+  inline void set(const uint32_t num){
+    uint32_t &value = _table[_hash1(num)];
+    if(value == 0){
+      value = num;
+    }else{
+      if(value != num){
+	_stlHash.insert(num);
+      }
+    }
+  }
+  
+  inline void insert(const uint32_t num){
+    set(num);
+  }
+
+  inline void reset(const uint32_t num){
+    const uint32_t hashValue = _hash1(num);
+    if(_table[hashValue] != 0){
+      if(_table[hashValue] != num){
+	_stlHash.erase(num);
+      }else{
+	_table[hashValue] = UINT_MAX;
+      }
+    }
+  }
+};
+

core/src/index/thirdparty/NGT/lib/NGT/MmapManager.h

+//
+// Copyright (C) 2015-2020 Yahoo Japan Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#pragma once
+
+#include <sys/types.h>
+#include <stdint.h>
+#include <string>
+#include <memory>
+
+#define ABS_ADDR(x, y) (void *)(x + (char *)y);
+
+#define USE_MMAP_MANAGER
+
+namespace MemoryManager{
+  
+  typedef enum _option_reuse_t{
+    REUSE_DATA_CLASSIFY,
+    REUSE_DATA_QUEUE,
+    REUSE_DATA_QUEUE_PLUS,    
+  }option_reuse_t;
+
+  typedef enum _reuse_state_t{
+    REUSE_STATE_OK, 
+    REUSE_STATE_FALSE, 
+    REUSE_STATE_ALLOC, 
+  }reuse_state_t;
+
+  typedef enum _check_statistics_t{
+    CHECK_STATS_USE_SIZE,  
+    CHECK_STATS_USE_NUM,   
+    CHECK_STATS_FREE_SIZE, 
+    CHECK_STATS_FREE_NUM,  
+  }check_statistics_t;
+
+  typedef struct _init_option_st{
+    bool use_expand;  
+    option_reuse_t reuse_type; 
+  }init_option_st;
+
+
+  class MmapManager{
+  public:
+    MmapManager();
+    ~MmapManager();
+        
+    bool init(const std::string &filePath, size_t size, const init_option_st *optionst = NULL) const;
+    bool openMemory(const std::string &filePath);
+    void closeMemory(const bool force = false);
+    off_t alloc(const size_t size, const bool not_reuse_flag = false);
+    void free(const off_t p);
+    off_t reuse(const size_t size, reuse_state_t &reuse_state);
+    void *getAbsAddr(off_t p) const;
+    off_t getRelAddr(const void *p) const;
+
+    size_t getTotalSize() const; 
+    size_t getUseSize() const;   
+    uint64_t getUseNum() const;  
+    size_t getFreeSize() const;  
+    uint64_t getFreeNum() const; 
+    uint16_t getUnitNum() const; 
+    size_t getQueueCapacity() const; 
+    uint64_t getQueueNum() const; 
+    uint64_t getLargeListNum() const;    
+
+    void dumpHeap() const;
+
+    bool isOpen() const;
+    void *getEntryHook() const;
+    void setEntryHook(const void *entry_p);
+
+    // static method --- 
+    static void setDefaultOptionValue(init_option_st &optionst);
+    static size_t getAlignSize(size_t size);
+
+  private:
+    class Impl;
+    std::unique_ptr<Impl> _impl;
+  };
+
+  std::string getErrorStr(int32_t err_num);
+}

core/src/index/thirdparty/NGT/lib/NGT/MmapManagerException.h

+//
+// Copyright (C) 2015-2020 Yahoo Japan Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#pragma once
+
+#include <iostream>
+#include <exception>
+#include <stdexcept>
+
+namespace MemoryManager{
+  class MmapManagerException : public std::domain_error{
+  public:
+  MmapManagerException(const std::string &msg) : std::domain_error(msg){}
+  };
+}