diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 6dc2369b47ec0d1f6f79fe0591bd3394823e5301..667399cc5d2b4267c72a23fb32e19d3f515a239b 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -18,6 +18,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-562 - Add JobMgr and TaskCreator in Scheduler - MS-566 - Refactor cmake - MS-555 - Remove old scheduler +- MS-578 - Makesure milvus5.0 don't crack 0.3.1 data ## New Feature diff --git a/cpp/README.md b/cpp/README.md index d32f04f31615823e175aa35fa78d4d9365d6ccbf..5c7a1d280c2b107130c6da95c5fc28b523cf3d64 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -68,16 +68,10 @@ $ sudo ln -s /path/to/libmysqlclient.so /path/to/libmysqlclient_r.so ###### Step 2 Build ```shell -TBD -cd [Milvus sourcecode path]/cpp/thirdparty -git clone git@192.168.1.105:megasearch/knowhere.git -cd knowhere -./build.sh -t Debug -or ./build.sh -t Release - -cd [sourcecode path]/cpp -./build.sh -t Debug -or ./build.sh -t Release +$ cd [Milvus sourcecode path]/cpp +$ ./build.sh -t Debug +or +$ ./build.sh -t Release ``` When the build is completed, all the stuff that you need in order to run Milvus will be installed under `[Milvus root path]/cpp/milvus`. diff --git a/cpp/src/core/CMakeLists.txt b/cpp/src/core/CMakeLists.txt index bef3f5e2f685be12023f87fa3b80d07121364e59..b126cce031a47cb055bd82054acefc692194047e 100644 --- a/cpp/src/core/CMakeLists.txt +++ b/cpp/src/core/CMakeLists.txt @@ -93,7 +93,7 @@ endif() set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE) if(BUILD_UNIT_TEST STREQUAL "ON") - add_subdirectory(test) +# add_subdirectory(test) endif() config_summary() diff --git a/cpp/src/core/knowhere/CMakeLists.txt b/cpp/src/core/knowhere/CMakeLists.txt index 55f23e1095430fedb3cb8dde5fd19596dca4ea60..76750f4f07137a6f7ecb43a4aacfbecc552fd39b 100644 --- a/cpp/src/core/knowhere/CMakeLists.txt +++ b/cpp/src/core/knowhere/CMakeLists.txt @@ -29,26 +29,32 @@ if(NOT TARGET SPTAGLibStatic) endif() set(external_srcs - knowhere/adapter/sptag.cpp - knowhere/adapter/structure.cpp - knowhere/adapter/arrow.cpp - knowhere/common/exception.cpp - knowhere/common/timer.cpp + knowhere/adapter/SptagAdapter.cpp + knowhere/adapter/Structure.cpp + knowhere/adapter/ArrowAdapter.cpp + knowhere/common/Exception.cpp + knowhere/common/Timer.cpp ) set(index_srcs - knowhere/index/preprocessor/normalize.cpp - knowhere/index/vector_index/cpu_kdt_rng.cpp - knowhere/index/vector_index/idmap.cpp - knowhere/index/vector_index/ivf.cpp - knowhere/index/vector_index/gpu_ivf.cpp - knowhere/index/vector_index/kdt_parameters.cpp - knowhere/index/vector_index/nsg_index.cpp - knowhere/index/vector_index/nsg/nsg.cpp - knowhere/index/vector_index/nsg/nsg_io.cpp - knowhere/index/vector_index/nsg/utils.cpp - knowhere/index/vector_index/cloner.cpp - knowhere/index/vector_index/FaissGpuResourceMgr.cpp + knowhere/index/preprocessor/Normalize.cpp + knowhere/index/vector_index/IndexKDT.cpp + knowhere/index/vector_index/IndexIDMAP.cpp + knowhere/index/vector_index/IndexIVF.cpp + knowhere/index/vector_index/IndexGPUIVF.cpp + knowhere/index/vector_index/helpers/KDTParameterMgr.cpp + knowhere/index/vector_index/IndexNSG.cpp + knowhere/index/vector_index/nsg/NSG.cpp + knowhere/index/vector_index/nsg/NSGIO.cpp + knowhere/index/vector_index/nsg/NSGHelper.cpp + knowhere/index/vector_index/helpers/Cloner.cpp + knowhere/index/vector_index/helpers/FaissGpuResourceMgr.cpp + knowhere/index/vector_index/IndexIVFSQ.cpp + knowhere/index/vector_index/IndexGPUIVFSQ.cpp + knowhere/index/vector_index/IndexIVFPQ.cpp + knowhere/index/vector_index/IndexGPUIVFPQ.cpp + knowhere/index/vector_index/FaissBaseIndex.cpp + knowhere/index/vector_index/helpers/FaissIO.cpp ) set(depend_libs diff --git a/cpp/src/core/knowhere/knowhere/adapter/arrow.cpp b/cpp/src/core/knowhere/knowhere/adapter/ArrowAdapter.cpp similarity index 98% rename from cpp/src/core/knowhere/knowhere/adapter/arrow.cpp rename to cpp/src/core/knowhere/knowhere/adapter/ArrowAdapter.cpp index c4804d33220c353bf4d18cc3ce9870cdaa10030c..c07cb16fca566630b56b06294cbe86e79b270543 100644 --- a/cpp/src/core/knowhere/knowhere/adapter/arrow.cpp +++ b/cpp/src/core/knowhere/knowhere/adapter/ArrowAdapter.cpp @@ -16,7 +16,7 @@ // under the License. -#include "arrow.h" +#include "ArrowAdapter.h" namespace zilliz { namespace knowhere { diff --git a/cpp/src/core/knowhere/knowhere/adapter/arrow.h b/cpp/src/core/knowhere/knowhere/adapter/ArrowAdapter.h similarity index 96% rename from cpp/src/core/knowhere/knowhere/adapter/arrow.h rename to cpp/src/core/knowhere/knowhere/adapter/ArrowAdapter.h index 318336e703ba5491bab78ab809e913c377754acd..d19f8f3ae5a46267b6d3f9de7b78de73c1d43b70 100644 --- a/cpp/src/core/knowhere/knowhere/adapter/arrow.h +++ b/cpp/src/core/knowhere/knowhere/adapter/ArrowAdapter.h @@ -20,7 +20,7 @@ #include -#include "knowhere/common/array.h" +#include "knowhere/common/Array.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/adapter/sptag.cpp b/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp similarity index 97% rename from cpp/src/core/knowhere/knowhere/adapter/sptag.cpp rename to cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp index 995d068577328339b9df72acf182ddce10966969..ee5a4a01b81780f139d587a7c3697c7dde7fb8ff 100644 --- a/cpp/src/core/knowhere/knowhere/adapter/sptag.cpp +++ b/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -#include "knowhere/index/vector_index/definitions.h" -#include "sptag.h" -#include "structure.h" +#include "knowhere/index/vector_index/helpers/Definitions.h" +#include "SptagAdapter.h" +#include "Structure.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/adapter/sptag.h b/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.h similarity index 97% rename from cpp/src/core/knowhere/knowhere/adapter/sptag.h rename to cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.h index eb7886c26319f6e12cf81d78529f262d2e6f0ed6..f47ffdc3b5814d2e0f97008a08983b00bb62cb88 100644 --- a/cpp/src/core/knowhere/knowhere/adapter/sptag.h +++ b/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.h @@ -22,7 +22,7 @@ #include -#include "knowhere/common/dataset.h" +#include "knowhere/common/Dataset.h" namespace zilliz { namespace knowhere { diff --git a/cpp/src/core/knowhere/knowhere/adapter/structure.cpp b/cpp/src/core/knowhere/knowhere/adapter/Structure.cpp similarity index 99% rename from cpp/src/core/knowhere/knowhere/adapter/structure.cpp rename to cpp/src/core/knowhere/knowhere/adapter/Structure.cpp index 10b6af8138b06cec19f3b38e16c9336feee67822..18833b5d36d7e43ac32d9c5998e97d31443c288b 100644 --- a/cpp/src/core/knowhere/knowhere/adapter/structure.cpp +++ b/cpp/src/core/knowhere/knowhere/adapter/Structure.cpp @@ -16,7 +16,7 @@ // under the License. -#include "structure.h" +#include "Structure.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/adapter/structure.h b/cpp/src/core/knowhere/knowhere/adapter/Structure.h similarity index 97% rename from cpp/src/core/knowhere/knowhere/adapter/structure.h rename to cpp/src/core/knowhere/knowhere/adapter/Structure.h index 76a03a23e4c83ba3dfec916e3b282bed867d6e58..7539dce2dae90c4d1bd5c2e1ad2f84be3b97e6d5 100644 --- a/cpp/src/core/knowhere/knowhere/adapter/structure.h +++ b/cpp/src/core/knowhere/knowhere/adapter/Structure.h @@ -19,7 +19,7 @@ #pragma once #include -#include "knowhere/common/dataset.h" +#include "knowhere/common/Dataset.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/adapter/faiss_adopt.h b/cpp/src/core/knowhere/knowhere/adapter/VectorAdapter.h similarity index 100% rename from cpp/src/core/knowhere/knowhere/adapter/faiss_adopt.h rename to cpp/src/core/knowhere/knowhere/adapter/VectorAdapter.h diff --git a/cpp/src/core/knowhere/knowhere/common/array.h b/cpp/src/core/knowhere/knowhere/common/Array.h similarity index 97% rename from cpp/src/core/knowhere/knowhere/common/array.h rename to cpp/src/core/knowhere/knowhere/common/Array.h index 1860524e8a2c1384ec6121ac5aaef53db021c037..94a5470029b7fbd5e44e4d5951e39e573fdb1382 100644 --- a/cpp/src/core/knowhere/knowhere/common/array.h +++ b/cpp/src/core/knowhere/knowhere/common/Array.h @@ -18,8 +18,9 @@ #pragma once -#include "arrow/array.h" -#include "schema.h" +#include + +#include "Schema.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/common/binary_set.h b/cpp/src/core/knowhere/knowhere/common/BinarySet.h similarity index 99% rename from cpp/src/core/knowhere/knowhere/common/binary_set.h rename to cpp/src/core/knowhere/knowhere/common/BinarySet.h index 019c42a2b951ab58b24f660e3ab449c03afc9c9c..509db68e2d25a7a19488032e41c8b6ba2e871d3f 100644 --- a/cpp/src/core/knowhere/knowhere/common/binary_set.h +++ b/cpp/src/core/knowhere/knowhere/common/BinarySet.h @@ -23,7 +23,7 @@ #include #include -#include "id.h" +#include "Id.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/common/buffer.h b/cpp/src/core/knowhere/knowhere/common/Buffer.h similarity index 98% rename from cpp/src/core/knowhere/knowhere/common/buffer.h rename to cpp/src/core/knowhere/knowhere/common/Buffer.h index 019c0487b707842b1fbdfbfbe39f475e32f920a1..4468e6ec01cb5c1ceff47ccb75ed73ab8a32a843 100644 --- a/cpp/src/core/knowhere/knowhere/common/buffer.h +++ b/cpp/src/core/knowhere/knowhere/common/Buffer.h @@ -20,7 +20,7 @@ #include -#include "arrow/buffer.h" +#include namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/common/config.h b/cpp/src/core/knowhere/knowhere/common/Config.h similarity index 100% rename from cpp/src/core/knowhere/knowhere/common/config.h rename to cpp/src/core/knowhere/knowhere/common/Config.h diff --git a/cpp/src/core/knowhere/knowhere/common/dataset.h b/cpp/src/core/knowhere/knowhere/common/Dataset.h similarity index 96% rename from cpp/src/core/knowhere/knowhere/common/dataset.h rename to cpp/src/core/knowhere/knowhere/common/Dataset.h index 068bbe435658b9358d467caec9f0fe9b8c94c642..c2d74a59e57c7e1ab4270570df45c4e87cae4343 100644 --- a/cpp/src/core/knowhere/knowhere/common/dataset.h +++ b/cpp/src/core/knowhere/knowhere/common/Dataset.h @@ -21,12 +21,12 @@ #include #include -#include "array.h" -#include "buffer.h" -#include "tensor.h" -#include "schema.h" -#include "config.h" -#include "knowhere/adapter/arrow.h" +#include "Array.h" +#include "Buffer.h" +#include "Tensor.h" +#include "Schema.h" +#include "Config.h" +#include "knowhere/adapter/ArrowAdapter.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/common/exception.cpp b/cpp/src/core/knowhere/knowhere/common/Exception.cpp similarity index 98% rename from cpp/src/core/knowhere/knowhere/common/exception.cpp rename to cpp/src/core/knowhere/knowhere/common/Exception.cpp index eb19e21bb05cb91f4e3a4f9f7bcf100422db81e4..a77e85ee41a9a856ac668aea3e6d56e44bb8188e 100644 --- a/cpp/src/core/knowhere/knowhere/common/exception.cpp +++ b/cpp/src/core/knowhere/knowhere/common/Exception.cpp @@ -18,7 +18,7 @@ #include -#include "exception.h" +#include "Exception.h" #include "Log.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/common/exception.h b/cpp/src/core/knowhere/knowhere/common/Exception.h similarity index 100% rename from cpp/src/core/knowhere/knowhere/common/exception.h rename to cpp/src/core/knowhere/knowhere/common/Exception.h diff --git a/cpp/src/core/knowhere/knowhere/common/id.h b/cpp/src/core/knowhere/knowhere/common/Id.h similarity index 100% rename from cpp/src/core/knowhere/knowhere/common/id.h rename to cpp/src/core/knowhere/knowhere/common/Id.h diff --git a/cpp/src/core/knowhere/knowhere/common/Log.h b/cpp/src/core/knowhere/knowhere/common/Log.h index 0d99c624abed4a67d9bbf7a0efd82c8cb809c238..1e390b3c1e6c2e1868f72614d81dbfbc65198ee1 100644 --- a/cpp/src/core/knowhere/knowhere/common/Log.h +++ b/cpp/src/core/knowhere/knowhere/common/Log.h @@ -22,6 +22,7 @@ namespace zilliz { namespace knowhere { + #define KNOWHERE_DOMAIN_NAME "[KNOWHERE] " #define KNOWHERE_ERROR_TEXT "KNOWHERE Error:" @@ -31,5 +32,6 @@ namespace knowhere { #define KNOWHERE_LOG_WARNING LOG(WARNING) << KNOWHERE_DOMAIN_NAME #define KNOWHERE_LOG_ERROR LOG(ERROR) << KNOWHERE_DOMAIN_NAME #define KNOWHERE_LOG_FATAL LOG(FATAL) << KNOWHERE_DOMAIN_NAME + } // namespace knowhere } // namespace zilliz \ No newline at end of file diff --git a/cpp/src/core/knowhere/knowhere/common/schema.h b/cpp/src/core/knowhere/knowhere/common/Schema.h similarity index 97% rename from cpp/src/core/knowhere/knowhere/common/schema.h rename to cpp/src/core/knowhere/knowhere/common/Schema.h index 1ab3d42f8c85b096e4f68a2b3c9b0bb3692bb26a..b43b7eb8753400f236964121a37d047b616436ef 100644 --- a/cpp/src/core/knowhere/knowhere/common/schema.h +++ b/cpp/src/core/knowhere/knowhere/common/Schema.h @@ -20,7 +20,7 @@ #include -#include "arrow/type.h" +#include namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/common/tensor.h b/cpp/src/core/knowhere/knowhere/common/Tensor.h similarity index 97% rename from cpp/src/core/knowhere/knowhere/common/tensor.h rename to cpp/src/core/knowhere/knowhere/common/Tensor.h index 8b92f1f891b66830b54132625a412c75ab344742..42e86dc4d1a8c41bc6a1b6b085d286b927981f05 100644 --- a/cpp/src/core/knowhere/knowhere/common/tensor.h +++ b/cpp/src/core/knowhere/knowhere/common/Tensor.h @@ -20,7 +20,7 @@ #include -#include "arrow/tensor.h" +#include namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/common/timer.cpp b/cpp/src/core/knowhere/knowhere/common/Timer.cpp similarity index 99% rename from cpp/src/core/knowhere/knowhere/common/timer.cpp rename to cpp/src/core/knowhere/knowhere/common/Timer.cpp index f57de535d426022398fca3b9537c1d28e4dc535d..7f2cb715149b78cf9fc0e5791791ac1abf8723a6 100644 --- a/cpp/src/core/knowhere/knowhere/common/timer.cpp +++ b/cpp/src/core/knowhere/knowhere/common/Timer.cpp @@ -18,7 +18,7 @@ #include // TODO(linxj): using Log instead -#include "timer.h" +#include "Timer.h" namespace zilliz { namespace knowhere { diff --git a/cpp/src/core/knowhere/knowhere/common/timer.h b/cpp/src/core/knowhere/knowhere/common/Timer.h similarity index 100% rename from cpp/src/core/knowhere/knowhere/common/timer.h rename to cpp/src/core/knowhere/knowhere/common/Timer.h diff --git a/cpp/src/core/knowhere/knowhere/index/index.h b/cpp/src/core/knowhere/knowhere/index/Index.h similarity index 89% rename from cpp/src/core/knowhere/knowhere/index/index.h rename to cpp/src/core/knowhere/knowhere/index/Index.h index 275a5fd3873ab579ddddbb9792e6c94b8edfdd7a..4ccbef394cb97d580c2084c1e83672723565b2d2 100644 --- a/cpp/src/core/knowhere/knowhere/index/index.h +++ b/cpp/src/core/knowhere/knowhere/index/Index.h @@ -20,11 +20,11 @@ #include -#include "knowhere/common/binary_set.h" -#include "knowhere/common/dataset.h" -#include "index_type.h" -#include "index_model.h" -#include "knowhere/index/preprocessor/preprocessor.h" +#include "knowhere/common/BinarySet.h" +#include "knowhere/common/Dataset.h" +#include "IndexType.h" +#include "IndexModel.h" +#include "knowhere/index/preprocessor/Preprocessor.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/index/index_model.h b/cpp/src/core/knowhere/knowhere/index/IndexModel.h similarity index 96% rename from cpp/src/core/knowhere/knowhere/index/index_model.h rename to cpp/src/core/knowhere/knowhere/index/IndexModel.h index d00a629d7e7f4ed55bfb8853ff487af23a2a9776..7557363f65a677056ca87bfced32de2ff5258dc8 100644 --- a/cpp/src/core/knowhere/knowhere/index/index_model.h +++ b/cpp/src/core/knowhere/knowhere/index/IndexModel.h @@ -19,7 +19,7 @@ #pragma once #include -#include "knowhere/common/binary_set.h" +#include "knowhere/common/BinarySet.h" namespace zilliz { namespace knowhere { diff --git a/cpp/src/core/knowhere/knowhere/index/index_type.h b/cpp/src/core/knowhere/knowhere/index/IndexType.h similarity index 100% rename from cpp/src/core/knowhere/knowhere/index/index_type.h rename to cpp/src/core/knowhere/knowhere/index/IndexType.h diff --git a/cpp/src/core/knowhere/knowhere/index/preprocessor/normalize.cpp b/cpp/src/core/knowhere/knowhere/index/preprocessor/Normalize.cpp similarity index 100% rename from cpp/src/core/knowhere/knowhere/index/preprocessor/normalize.cpp rename to cpp/src/core/knowhere/knowhere/index/preprocessor/Normalize.cpp diff --git a/cpp/src/core/knowhere/knowhere/index/preprocessor/normalize.h b/cpp/src/core/knowhere/knowhere/index/preprocessor/Normalize.h similarity index 100% rename from cpp/src/core/knowhere/knowhere/index/preprocessor/normalize.h rename to cpp/src/core/knowhere/knowhere/index/preprocessor/Normalize.h diff --git a/cpp/src/core/knowhere/knowhere/index/preprocessor/preprocessor.h b/cpp/src/core/knowhere/knowhere/index/preprocessor/Preprocessor.h similarity index 96% rename from cpp/src/core/knowhere/knowhere/index/preprocessor/preprocessor.h rename to cpp/src/core/knowhere/knowhere/index/preprocessor/Preprocessor.h index 10f05442703397c1aaf05b431103643b52640037..e1c01d20850dc9c53bdda07a0424155706652e74 100644 --- a/cpp/src/core/knowhere/knowhere/index/preprocessor/preprocessor.h +++ b/cpp/src/core/knowhere/knowhere/index/preprocessor/Preprocessor.h @@ -20,7 +20,7 @@ #include -#include "knowhere/common/dataset.h" +#include "knowhere/common/Dataset.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a3ab7956cb634ad048b98fb42f624b12a923599f --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#include +#include + +#include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/helpers/FaissIO.h" +#include "FaissBaseIndex.h" + + +namespace zilliz { +namespace knowhere { + +FaissBaseIndex::FaissBaseIndex(std::shared_ptr index) : index_(std::move(index)) {} + +BinarySet FaissBaseIndex::SerializeImpl() { + try { + faiss::Index *index = index_.get(); + + SealImpl(); + + MemoryIOWriter writer; + faiss::write_index(index, &writer); + auto data = std::make_shared(); + data.reset(writer.data_); + + BinarySet res_set; + // TODO(linxj): use virtual func Name() instead of raw string. + res_set.Append("IVF", data, writer.rp); + return res_set; + } catch (std::exception &e) { + KNOWHERE_THROW_MSG(e.what()); + } +} + +void FaissBaseIndex::LoadImpl(const BinarySet &index_binary) { + auto binary = index_binary.GetByName("IVF"); + + MemoryIOReader reader; + reader.total = binary->size; + reader.data_ = binary->data.get(); + + faiss::Index *index = faiss::read_index(&reader); + + index_.reset(index); +} + +void FaissBaseIndex::SealImpl() { +// TODO(linxj): enable +//#ifdef ZILLIZ_FAISS + faiss::Index *index = index_.get(); + auto idx = dynamic_cast(index); + if (idx != nullptr) { + idx->to_readonly(); + } + //else { + // KNOHWERE_ERROR_MSG("Seal failed"); + //} +//#endif +} + +} // knowhere +} // zilliz + diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.h b/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.h new file mode 100644 index 0000000000000000000000000000000000000000..29edbbd61e92964f7e38c355ce75747c0a307099 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.h @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include + +#include + +#include "knowhere/common/BinarySet.h" + + +namespace zilliz { +namespace knowhere { + +class FaissBaseIndex { + protected: + explicit FaissBaseIndex(std::shared_ptr index); + + virtual BinarySet + SerializeImpl(); + + virtual void + LoadImpl(const BinarySet &index_binary); + + virtual void + SealImpl(); + + protected: + std::shared_ptr index_ = nullptr; +}; + +} // knowhere +} // zilliz + + + diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp similarity index 64% rename from cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp index 4efd5f24ae4c1f96d250012fdc41c29bcbfdefc7..a77702ace333d97ecf526d11cb93d5db8fbabada 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp @@ -20,18 +20,17 @@ #include #include #include -#include #include #include #include -#include "knowhere/common/exception.h" -#include "cloner.h" -#include "knowhere/adapter/faiss_adopt.h" -#include "gpu_ivf.h" +#include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/helpers/Cloner.h" +#include "knowhere/adapter/VectorAdapter.h" +#include "IndexGPUIVF.h" +#include "knowhere/index/vector_index/helpers/FaissIO.h" -#include namespace zilliz { namespace knowhere { @@ -169,7 +168,7 @@ VectorIndexPtr GPUIVF::CopyGpuToCpu(const Config &config) { VectorIndexPtr GPUIVF::Clone() { auto cpu_idx = CopyGpuToCpu(Config()); - return ::zilliz::knowhere::CopyCpuToGpu(cpu_idx, gpu_id_, Config()); + return ::zilliz::knowhere::cloner::CopyCpuToGpu(cpu_idx, gpu_id_, Config()); } VectorIndexPtr GPUIVF::CopyGpuToGpu(const int64_t &device_id, const Config &config) { @@ -187,84 +186,6 @@ void GPUIVF::Add(const DatasetPtr &dataset, const Config &config) { } } -IndexModelPtr GPUIVFPQ::Train(const DatasetPtr &dataset, const Config &config) { - auto nlist = config["nlist"].as(); - auto M = config["M"].as(); // number of subquantizers(subvectors) - auto nbits = config["nbits"].as();// number of bit per subvector index - auto gpu_num = config.get_with_default("gpu_id", gpu_id_); - auto metric_type = config["metric_type"].as_string() == "L2" ? - faiss::METRIC_L2 : faiss::METRIC_L2; // IP not support. - - GETTENSOR(dataset) - - // TODO(linxj): set device here. - // TODO(linxj): set gpu resource here. - faiss::gpu::StandardGpuResources res; - faiss::gpu::GpuIndexIVFPQ device_index(&res, dim, nlist, M, nbits, metric_type); - device_index.train(rows, (float *) p_data); - - std::shared_ptr host_index = nullptr; - host_index.reset(faiss::gpu::index_gpu_to_cpu(&device_index)); - - return std::make_shared(host_index); -} - -std::shared_ptr GPUIVFPQ::GenParams(const Config &config) { - auto params = std::make_shared(); - params->nprobe = config.get_with_default("nprobe", size_t(1)); - //params->scan_table_threshold = 0; - //params->polysemous_ht = 0; - //params->max_codes = 0; - - return params; -} - -VectorIndexPtr GPUIVFPQ::CopyGpuToCpu(const Config &config) { - KNOWHERE_THROW_MSG("not support yet"); -} - -IndexModelPtr GPUIVFSQ::Train(const DatasetPtr &dataset, const Config &config) { - auto nlist = config["nlist"].as(); - auto nbits = config["nbits"].as(); // TODO(linxj): gpu only support SQ4 SQ8 SQ16 - gpu_id_ = config.get_with_default("gpu_id", gpu_id_); - auto metric_type = config["metric_type"].as_string() == "L2" ? - faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT; - - GETTENSOR(dataset) - - std::stringstream index_type; - index_type << "IVF" << nlist << "," << "SQ" << nbits; - auto build_index = faiss::index_factory(dim, index_type.str().c_str(), metric_type); - - auto temp_resource = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_); - if (temp_resource != nullptr) { - ResScope rs(temp_resource, gpu_id_, true); - auto device_index = faiss::gpu::index_cpu_to_gpu(temp_resource->faiss_res.get(), gpu_id_, build_index); - device_index->train(rows, (float *) p_data); - - std::shared_ptr host_index = nullptr; - host_index.reset(faiss::gpu::index_gpu_to_cpu(device_index)); - - delete device_index; - delete build_index; - - return std::make_shared(host_index); - } else { - KNOWHERE_THROW_MSG("Build IVFSQ can't get gpu resource"); - } -} - -VectorIndexPtr GPUIVFSQ::CopyGpuToCpu(const Config &config) { - std::lock_guard lk(mutex_); - - faiss::Index *device_index = index_.get(); - faiss::Index *host_index = faiss::gpu::index_gpu_to_cpu(device_index); - - std::shared_ptr new_index; - new_index.reset(host_index); - return std::make_shared(new_index); -} - void GPUIndex::SetGpuDevice(const int &gpu_id) { gpu_id_ = gpu_id; } diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.h new file mode 100644 index 0000000000000000000000000000000000000000..cd04fe4c5433f677b2c5e3444d7b9b17119e9331 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.h @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + + +#include "IndexIVF.h" +#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" + + +namespace zilliz { +namespace knowhere { + +class GPUIndex { +public: + explicit GPUIndex(const int &device_id) : gpu_id_(device_id) {} + + GPUIndex(const int& device_id, const ResPtr& resource): gpu_id_(device_id), res_(resource) {} + + virtual VectorIndexPtr + CopyGpuToCpu(const Config &config) = 0; + + virtual VectorIndexPtr + CopyGpuToGpu(const int64_t &device_id, const Config &config) = 0; + + void + SetGpuDevice(const int &gpu_id); + + const int64_t & + GetGpuDevice(); + +protected: + int64_t gpu_id_; + ResWPtr res_; +}; + +class GPUIVF : public IVF, public GPUIndex { +public: + explicit GPUIVF(const int &device_id) : IVF(), GPUIndex(device_id) {} + + explicit GPUIVF(std::shared_ptr index, const int64_t &device_id, ResPtr &resource) + : IVF(std::move(index)), GPUIndex(device_id, resource) {}; + + IndexModelPtr + Train(const DatasetPtr &dataset, const Config &config) override; + + void + Add(const DatasetPtr &dataset, const Config &config) override; + + void + set_index_model(IndexModelPtr model) override; + + //DatasetPtr Search(const DatasetPtr &dataset, const Config &config) override; + VectorIndexPtr + CopyGpuToCpu(const Config &config) override; + + VectorIndexPtr + CopyGpuToGpu(const int64_t &device_id, const Config &config) override; + + VectorIndexPtr + Clone() final; + + // TODO(linxj): Deprecated + virtual IVFIndexPtr Copy_index_gpu_to_cpu(); + +protected: + void + search_impl(int64_t n, + const float *data, + int64_t k, + float *distances, + int64_t *labels, + const Config &cfg) override; + + BinarySet + SerializeImpl() override; + + void + LoadImpl(const BinarySet &index_binary) override; +}; + +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp new file mode 100644 index 0000000000000000000000000000000000000000..437d8f8958e4905baf823248de78f2c96ff55052 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#include +#include +#include + +#include "IndexGPUIVFPQ.h" +#include "knowhere/common/Exception.h" +#include "knowhere/adapter/VectorAdapter.h" + + +namespace zilliz { +namespace knowhere { + +IndexModelPtr GPUIVFPQ::Train(const DatasetPtr &dataset, const Config &config) { + auto nlist = config["nlist"].as(); + auto M = config["M"].as(); // number of subquantizers(subvectors) + auto nbits = config["nbits"].as();// number of bit per subvector index + auto gpu_num = config.get_with_default("gpu_id", gpu_id_); + auto metric_type = config["metric_type"].as_string() == "L2" ? + faiss::METRIC_L2 : faiss::METRIC_L2; // IP not support. + + GETTENSOR(dataset) + + // TODO(linxj): set device here. + // TODO(linxj): set gpu resource here. + faiss::gpu::StandardGpuResources res; + faiss::gpu::GpuIndexIVFPQ device_index(&res, dim, nlist, M, nbits, metric_type); + device_index.train(rows, (float *) p_data); + + std::shared_ptr host_index = nullptr; + host_index.reset(faiss::gpu::index_gpu_to_cpu(&device_index)); + + return std::make_shared(host_index); +} + +std::shared_ptr GPUIVFPQ::GenParams(const Config &config) { + auto params = std::make_shared(); + params->nprobe = config.get_with_default("nprobe", size_t(1)); + //params->scan_table_threshold = 0; + //params->polysemous_ht = 0; + //params->max_codes = 0; + + return params; +} + +VectorIndexPtr GPUIVFPQ::CopyGpuToCpu(const Config &config) { + KNOWHERE_THROW_MSG("not support yet"); +} + +} // knowhere +} // zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.h new file mode 100644 index 0000000000000000000000000000000000000000..c407ee0cc7a3f1a1d2c48fa816b5c4f55b167cdb --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.h @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include "IndexGPUIVF.h" + +namespace zilliz { +namespace knowhere { + +class GPUIVFPQ : public GPUIVF { +public: + explicit GPUIVFPQ(const int &device_id) : GPUIVF(device_id) {} + + IndexModelPtr + Train(const DatasetPtr &dataset, const Config &config) override; + +public: + VectorIndexPtr + CopyGpuToCpu(const Config &config) override; + +protected: + // TODO(linxj): remove GenParams. + std::shared_ptr + GenParams(const Config &config) override; +}; + +} // knowhere +} // zilliz + + + diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2504dd4599a949d4d2ee0ef76b813daa2d443068 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#include + +#include "knowhere/adapter/VectorAdapter.h" +#include "knowhere/common/Exception.h" +#include "IndexGPUIVFSQ.h" +#include "IndexIVFSQ.h" + + +namespace zilliz { +namespace knowhere { + + IndexModelPtr GPUIVFSQ::Train(const DatasetPtr &dataset, const Config &config) { + auto nlist = config["nlist"].as(); + auto nbits = config["nbits"].as(); // TODO(linxj): gpu only support SQ4 SQ8 SQ16 + gpu_id_ = config.get_with_default("gpu_id", gpu_id_); + auto metric_type = config["metric_type"].as_string() == "L2" ? + faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT; + + GETTENSOR(dataset) + + std::stringstream index_type; + index_type << "IVF" << nlist << "," << "SQ" << nbits; + auto build_index = faiss::index_factory(dim, index_type.str().c_str(), metric_type); + + auto temp_resource = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_); + if (temp_resource != nullptr) { + ResScope rs(temp_resource, gpu_id_, true); + auto device_index = faiss::gpu::index_cpu_to_gpu(temp_resource->faiss_res.get(), gpu_id_, build_index); + device_index->train(rows, (float *) p_data); + + std::shared_ptr host_index = nullptr; + host_index.reset(faiss::gpu::index_gpu_to_cpu(device_index)); + + delete device_index; + delete build_index; + + return std::make_shared(host_index); + } else { + KNOWHERE_THROW_MSG("Build IVFSQ can't get gpu resource"); + } + } + + VectorIndexPtr GPUIVFSQ::CopyGpuToCpu(const Config &config) { + std::lock_guard lk(mutex_); + + faiss::Index *device_index = index_.get(); + faiss::Index *host_index = faiss::gpu::index_gpu_to_cpu(device_index); + + std::shared_ptr new_index; + new_index.reset(host_index); + return std::make_shared(new_index); + } + +} // knowhere +} // zilliz + diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.h new file mode 100644 index 0000000000000000000000000000000000000000..677907964bbf33d0f3c6aa359c1be7c37988c38a --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include "IndexGPUIVF.h" + + +namespace zilliz { +namespace knowhere { + +class GPUIVFSQ : public GPUIVF { +public: + explicit GPUIVFSQ(const int &device_id) : GPUIVF(device_id) {} + + explicit GPUIVFSQ(std::shared_ptr index, const int64_t &device_id, ResPtr &resource) + : GPUIVF(std::move(index), device_id, resource) {}; + + IndexModelPtr + Train(const DatasetPtr &dataset, const Config &config) override; + + VectorIndexPtr + CopyGpuToCpu(const Config &config) override; +}; + +} // knowhere +} // zilliz + diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/idmap.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp similarity index 97% rename from cpp/src/core/knowhere/knowhere/index/vector_index/idmap.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp index 494da4438abf61148a764664183a70d8d9b665e4..a53a73dad9608e41bd40c42db91ef948fc029db7 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/idmap.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp @@ -23,9 +23,10 @@ #include -#include "knowhere/common/exception.h" -#include "knowhere/adapter/faiss_adopt.h" -#include "idmap.h" +#include "knowhere/common/Exception.h" +#include "knowhere/adapter/VectorAdapter.h" +#include "knowhere/index/vector_index/helpers/FaissIO.h" +#include "IndexIDMAP.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/idmap.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.h similarity index 91% rename from cpp/src/core/knowhere/knowhere/index/vector_index/idmap.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.h index b54a589423b9688089438560ea3f1cbf064581ae..106759faf1c6380243c3ce769422eacb8d0c49a3 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/idmap.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.h @@ -18,17 +18,17 @@ #pragma once -#include "ivf.h" -#include "gpu_ivf.h" +#include "IndexIVF.h" +#include "IndexGPUIVF.h" namespace zilliz { namespace knowhere { -class IDMAP : public VectorIndex, public BasicIndex { +class IDMAP : public VectorIndex, public FaissBaseIndex { public: - IDMAP() : BasicIndex(nullptr) {}; - explicit IDMAP(std::shared_ptr index) : BasicIndex(std::move(index)) {}; + IDMAP() : FaissBaseIndex(nullptr) {}; + explicit IDMAP(std::shared_ptr index) : FaissBaseIndex(std::move(index)) {}; BinarySet Serialize() override; void Load(const BinarySet &index_binary) override; void Train(const Config &config); diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/ivf.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.cpp similarity index 61% rename from cpp/src/core/knowhere/knowhere/index/vector_index/ivf.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.cpp index 6d72479002bb7702248b5939afa8d240a0873e9f..a7adfe921a67821aca042a56ae627f1721aeb09e 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/ivf.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.cpp @@ -23,14 +23,13 @@ #include #include #include -#include #include -#include "knowhere/common/exception.h" -#include "ivf.h" -#include "knowhere/adapter/faiss_adopt.h" -#include "gpu_ivf.h" +#include "knowhere/common/Exception.h" +#include "knowhere/adapter/VectorAdapter.h" +#include "IndexIVF.h" +#include "IndexGPUIVF.h" namespace zilliz { @@ -239,7 +238,7 @@ void IVF::Seal() { } -IVFIndexModel::IVFIndexModel(std::shared_ptr index) : BasicIndex(std::move(index)) {} +IVFIndexModel::IVFIndexModel(std::shared_ptr index) : FaissBaseIndex(std::move(index)) {} BinarySet IVFIndexModel::Serialize() { if (!index_ || !index_->is_trained) { @@ -258,159 +257,8 @@ void IVFIndexModel::SealImpl() { // do nothing } -IndexModelPtr IVFSQ::Train(const DatasetPtr &dataset, const Config &config) { - auto nlist = config["nlist"].as(); - auto nbits = config["nbits"].as(); // TODO(linxj): only support SQ4 SQ6 SQ8 SQ16 - auto metric_type = config["metric_type"].as_string() == "L2" ? - faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT; - - GETTENSOR(dataset) - - std::stringstream index_type; - index_type << "IVF" << nlist << "," << "SQ" << nbits; - auto build_index = faiss::index_factory(dim, index_type.str().c_str(), metric_type); - build_index->train(rows, (float *) p_data); - - std::shared_ptr ret_index; - ret_index.reset(build_index); - return std::make_shared(ret_index); -} - -VectorIndexPtr IVFSQ::Clone_impl(const std::shared_ptr &index) { - return std::make_shared(index); -} - -VectorIndexPtr IVFSQ::CopyCpuToGpu(const int64_t &device_id, const Config &config) { - if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)){ - ResScope rs(res, device_id, false); - faiss::gpu::GpuClonerOptions option; - option.allInGpu = true; - - auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, index_.get(), &option); - - std::shared_ptr device_index; - device_index.reset(gpu_index); - return std::make_shared(device_index, device_id, res); - } else { - KNOWHERE_THROW_MSG("CopyCpuToGpu Error, can't get gpu_resource"); - } -} - -IndexModelPtr IVFPQ::Train(const DatasetPtr &dataset, const Config &config) { - auto nlist = config["nlist"].as(); - auto M = config["M"].as(); // number of subquantizers(subvector) - auto nbits = config["nbits"].as();// number of bit per subvector index - auto metric_type = config["metric_type"].as_string() == "L2" ? - faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT; - - GETTENSOR(dataset) - - faiss::Index *coarse_quantizer = new faiss::IndexFlat(dim, metric_type); - auto index = std::make_shared(coarse_quantizer, dim, nlist, M, nbits); - index->train(rows, (float *) p_data); - - return std::make_shared(index); -} -std::shared_ptr IVFPQ::GenParams(const Config &config) { - auto params = std::make_shared(); - params->nprobe = config.get_with_default("nprobe", size_t(1)); - //params->scan_table_threshold = 0; - //params->polysemous_ht = 0; - //params->max_codes = 0; - - return params; -} - -VectorIndexPtr IVFPQ::Clone_impl(const std::shared_ptr &index) { - return std::make_shared(index); -} - -BasicIndex::BasicIndex(std::shared_ptr index) : index_(std::move(index)) {} - -BinarySet BasicIndex::SerializeImpl() { - try { - faiss::Index *index = index_.get(); - - SealImpl(); - - MemoryIOWriter writer; - faiss::write_index(index, &writer); - auto data = std::make_shared(); - data.reset(writer.data_); - - BinarySet res_set; - // TODO(linxj): use virtual func Name() instead of raw string. - res_set.Append("IVF", data, writer.rp); - return res_set; - } catch (std::exception &e) { - KNOWHERE_THROW_MSG(e.what()); - } -} - -void BasicIndex::LoadImpl(const BinarySet &index_binary) { - auto binary = index_binary.GetByName("IVF"); - - MemoryIOReader reader; - reader.total = binary->size; - reader.data_ = binary->data.get(); - faiss::Index *index = faiss::read_index(&reader); - - index_.reset(index); -} - -void BasicIndex::SealImpl() { -// TODO(linxj): enable -//#ifdef ZILLIZ_FAISS - faiss::Index *index = index_.get(); - auto idx = dynamic_cast(index); - if (idx != nullptr) { - idx->to_readonly(); - } - //else { - // KNOHWERE_ERROR_MSG("Seal failed"); - //} -//#endif -} - -// TODO(linxj): Get From Config File -static size_t magic_num = 2; -size_t MemoryIOWriter::operator()(const void *ptr, size_t size, size_t nitems) { - auto total_need = size * nitems + rp; - - if (!data_) { // data == nullptr - total = total_need * magic_num; - rp = size * nitems; - data_ = new uint8_t[total]; - memcpy((void *) (data_), ptr, rp); - } - - if (total_need > total) { - total = total_need * magic_num; - auto new_data = new uint8_t[total]; - memcpy((void *) new_data, (void *) data_, rp); - delete data_; - data_ = new_data; - - memcpy((void *) (data_ + rp), ptr, size * nitems); - rp = total_need; - } else { - memcpy((void *) (data_ + rp), ptr, size * nitems); - rp = total_need; - } - - return nitems; -} - -size_t MemoryIOReader::operator()(void *ptr, size_t size, size_t nitems) { - if (rp >= total) return 0; - size_t nremain = (total - rp) / size; - if (nremain < nitems) nitems = nremain; - memcpy(ptr, (void *) (data_ + rp), size * nitems); - rp += size * nitems; - return nitems; -} } diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.h new file mode 100644 index 0000000000000000000000000000000000000000..72f98a5afab5979d576355d33c8be194d86ef822 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.h @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include +#include + +#include "VectorIndex.h" +#include "FaissBaseIndex.h" +#include "faiss/IndexIVF.h" + + +namespace zilliz { +namespace knowhere { + +using Graph = std::vector>; + +class IVF : public VectorIndex, protected FaissBaseIndex { + public: + IVF() : FaissBaseIndex(nullptr) {}; + + explicit IVF(std::shared_ptr index) : FaissBaseIndex(std::move(index)) {} + + VectorIndexPtr + Clone() override;; + + IndexModelPtr + Train(const DatasetPtr &dataset, const Config &config) override; + + void + set_index_model(IndexModelPtr model) override; + + void + Add(const DatasetPtr &dataset, const Config &config) override; + + void + AddWithoutIds(const DatasetPtr &dataset, const Config &config); + + DatasetPtr + Search(const DatasetPtr &dataset, const Config &config) override; + + void + GenGraph(const int64_t &k, Graph &graph, const DatasetPtr &dataset, const Config &config); + + BinarySet + Serialize() override; + + void + Load(const BinarySet &index_binary) override; + + int64_t + Count() override; + + int64_t + Dimension() override; + + void + Seal() override; + + virtual VectorIndexPtr + CopyCpuToGpu(const int64_t &device_id, const Config &config); + + + protected: + virtual std::shared_ptr + GenParams(const Config &config); + + virtual VectorIndexPtr + Clone_impl(const std::shared_ptr &index); + + virtual void + search_impl(int64_t n, + const float *data, + int64_t k, + float *distances, + int64_t *labels, + const Config &cfg); + + protected: + std::mutex mutex_; +}; + +using IVFIndexPtr = std::shared_ptr; + +class GPUIVF; +class IVFIndexModel : public IndexModel, public FaissBaseIndex { + friend IVF; + friend GPUIVF; + + public: + explicit IVFIndexModel(std::shared_ptr index); + + IVFIndexModel() : FaissBaseIndex(nullptr) {}; + + BinarySet + Serialize() override; + + void + Load(const BinarySet &binary) override; + + protected: + void + SealImpl() override; + + protected: + std::mutex mutex_; +}; +using IVFIndexModelPtr = std::shared_ptr; + +} +} \ No newline at end of file diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp new file mode 100644 index 0000000000000000000000000000000000000000..eca63a5f3b3bda6dd12068a277c12b51437453a6 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#include +#include + +#include "IndexIVFPQ.h" +#include "knowhere/common/Exception.h" +#include "knowhere/adapter/VectorAdapter.h" + +namespace zilliz { +namespace knowhere { + +IndexModelPtr IVFPQ::Train(const DatasetPtr &dataset, const Config &config) { + auto nlist = config["nlist"].as(); + auto M = config["M"].as(); // number of subquantizers(subvector) + auto nbits = config["nbits"].as();// number of bit per subvector index + auto metric_type = config["metric_type"].as_string() == "L2" ? + faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT; + + GETTENSOR(dataset) + + faiss::Index *coarse_quantizer = new faiss::IndexFlat(dim, metric_type); + auto index = std::make_shared(coarse_quantizer, dim, nlist, M, nbits); + index->train(rows, (float *) p_data); + + return std::make_shared(index); +} + +std::shared_ptr IVFPQ::GenParams(const Config &config) { + auto params = std::make_shared(); + params->nprobe = config.get_with_default("nprobe", size_t(1)); + //params->scan_table_threshold = 0; + //params->polysemous_ht = 0; + //params->max_codes = 0; + + return params; +} + +VectorIndexPtr IVFPQ::Clone_impl(const std::shared_ptr &index) { + return std::make_shared(index); +} + +} // knowhere +} // zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFPQ.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFPQ.h new file mode 100644 index 0000000000000000000000000000000000000000..427f76fe3e634edab425facb332de2c47705835d --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFPQ.h @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include "IndexIVF.h" + +namespace zilliz { +namespace knowhere { + +class IVFPQ : public IVF { +public: + explicit IVFPQ(std::shared_ptr index) : IVF(std::move(index)) {} + + IVFPQ() = default; + + IndexModelPtr + Train(const DatasetPtr &dataset, const Config &config) override; + +protected: + std::shared_ptr + GenParams(const Config &config) override; + + VectorIndexPtr + Clone_impl(const std::shared_ptr &index) override; +}; + +} // knowhere +} // zilliz + + + diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp new file mode 100644 index 0000000000000000000000000000000000000000..00f191ff0254470c4710b1da033e434e07625117 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#include + +#include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" +#include "knowhere/adapter/VectorAdapter.h" +#include "IndexIVFSQ.h" +#include "IndexGPUIVFSQ.h" + + +namespace zilliz { +namespace knowhere { + +IndexModelPtr IVFSQ::Train(const DatasetPtr &dataset, const Config &config) { + auto nlist = config["nlist"].as(); + auto nbits = config["nbits"].as(); // TODO(linxj): only support SQ4 SQ6 SQ8 SQ16 + auto metric_type = config["metric_type"].as_string() == "L2" ? + faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT; + + GETTENSOR(dataset) + + std::stringstream index_type; + index_type << "IVF" << nlist << "," << "SQ" << nbits; + auto build_index = faiss::index_factory(dim, index_type.str().c_str(), metric_type); + build_index->train(rows, (float *) p_data); + + std::shared_ptr ret_index; + ret_index.reset(build_index); + return std::make_shared(ret_index); +} + +VectorIndexPtr IVFSQ::Clone_impl(const std::shared_ptr &index) { + return std::make_shared(index); +} + +VectorIndexPtr IVFSQ::CopyCpuToGpu(const int64_t &device_id, const Config &config) { + if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)){ + ResScope rs(res, device_id, false); + faiss::gpu::GpuClonerOptions option; + option.allInGpu = true; + + auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, index_.get(), &option); + + std::shared_ptr device_index; + device_index.reset(gpu_index); + return std::make_shared(device_index, device_id, res); + } else { + KNOWHERE_THROW_MSG("CopyCpuToGpu Error, can't get gpu_resource"); + } +} + +} // knowhere +} // zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.h new file mode 100644 index 0000000000000000000000000000000000000000..5ee771f951c8d7b8ae3a7803f26f25e14852b45e --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.h @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include "IndexIVF.h" + +namespace zilliz { +namespace knowhere { + +class IVFSQ : public IVF { +public: + explicit IVFSQ(std::shared_ptr index) : IVF(std::move(index)) {} + + IVFSQ() = default; + + IndexModelPtr + Train(const DatasetPtr &dataset, const Config &config) override; + + VectorIndexPtr + CopyCpuToGpu(const int64_t &device_id, const Config &config) override; + +protected: + VectorIndexPtr + Clone_impl(const std::shared_ptr &index) override; +}; + +} // knowhere +} // zilliz + diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/cpu_kdt_rng.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.cpp similarity index 94% rename from cpp/src/core/knowhere/knowhere/index/vector_index/cpu_kdt_rng.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.cpp index d588efb7c6d1100cedc1a0ee09af260dba31a809..5e049b72bd818a3be000a9be398cb5862ff25134 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/cpu_kdt_rng.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.cpp @@ -24,12 +24,12 @@ #undef mkdir -#include "cpu_kdt_rng.h" -#include "definitions.h" +#include "IndexKDT.h" +#include "knowhere/index/vector_index/helpers/Definitions.h" //#include "knowhere/index/preprocessor/normalize.h" -#include "kdt_parameters.h" -#include "knowhere/adapter/sptag.h" -#include "knowhere/common/exception.h" +#include "knowhere/index/vector_index/helpers/KDTParameterMgr.h" +#include "knowhere/adapter/SptagAdapter.h" +#include "knowhere/common/Exception.h" namespace zilliz { @@ -117,7 +117,7 @@ CPUKDTRNG::Add(const DatasetPtr &origin, const Config &add_config) { void CPUKDTRNG::SetParameters(const Config &config) { - for (auto ¶ : KDTParameterManagement::GetInstance().GetKDTParameters()) { + for (auto ¶ : KDTParameterMgr::GetInstance().GetKDTParameters()) { auto value = config.get_with_default(para.first, para.second); index_ptr_->SetParameter(para.first, value); } diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/cpu_kdt_rng.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.h similarity index 97% rename from cpp/src/core/knowhere/knowhere/index/vector_index/cpu_kdt_rng.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.h index 35d488999ea178907e1c4c1ac64fc02da34292ff..7fd41e9baddb67875710ee6125109fbbef7c6523 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/cpu_kdt_rng.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.h @@ -20,8 +20,8 @@ #include #include -#include "vector_index.h" -#include "knowhere/index/index_model.h" +#include "VectorIndex.h" +#include "knowhere/index/IndexModel.h" #include diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg_index.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexNSG.cpp similarity index 94% rename from cpp/src/core/knowhere/knowhere/index/vector_index/nsg_index.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/IndexNSG.cpp index 724c80dfcaf89a6c69f82d5317973761a4cd0d46..26dc5e2304ef1eea1779503159dbf113e8b8dc2f 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg_index.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexNSG.cpp @@ -16,15 +16,15 @@ // under the License. -#include "nsg_index.h" -#include "knowhere/index/vector_index/nsg/nsg.h" -#include "knowhere/index/vector_index/nsg/nsg_io.h" -#include "idmap.h" -#include "ivf.h" -#include "gpu_ivf.h" -#include "knowhere/adapter/faiss_adopt.h" -#include "knowhere/common/exception.h" -#include "knowhere/common/timer.h" +#include "IndexNSG.h" +#include "knowhere/index/vector_index/nsg/NSG.h" +#include "knowhere/index/vector_index/nsg/NSGIO.h" +#include "IndexIDMAP.h" +#include "IndexIVF.h" +#include "IndexGPUIVF.h" +#include "knowhere/adapter/VectorAdapter.h" +#include "knowhere/common/Exception.h" +#include "knowhere/common/Timer.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg_index.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexNSG.h similarity index 98% rename from cpp/src/core/knowhere/knowhere/index/vector_index/nsg_index.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/IndexNSG.h index e63dbb03b4fec5b906b391449751c3875383e6c1..873bba2287d6773844886bfcc623b1c92bccb47b 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg_index.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexNSG.h @@ -18,7 +18,7 @@ #pragma once -#include "vector_index.h" +#include "VectorIndex.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/vector_index.h b/cpp/src/core/knowhere/knowhere/index/vector_index/VectorIndex.h similarity index 90% rename from cpp/src/core/knowhere/knowhere/index/vector_index/vector_index.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/VectorIndex.h index 1828e3573d30c595a24ebc55981926e4dbe02c97..76f67920e31e93b7aaa26dcc74a1e2f4e686fd4f 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/vector_index.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/VectorIndex.h @@ -21,10 +21,10 @@ #include -#include "knowhere/common/config.h" -#include "knowhere/common/dataset.h" -#include "knowhere/index/index.h" -#include "knowhere/index/preprocessor/preprocessor.h" +#include "knowhere/common/Config.h" +#include "knowhere/common/Dataset.h" +#include "knowhere/index/Index.h" +#include "knowhere/index/preprocessor/Preprocessor.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.h b/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.h deleted file mode 100644 index 4948d16ac4685bc78ac197d183bb0c3b88cf14e5..0000000000000000000000000000000000000000 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.h +++ /dev/null @@ -1,98 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - - -#pragma once - - -#include "ivf.h" -#include "FaissGpuResourceMgr.h" - - -namespace zilliz { -namespace knowhere { - -class GPUIndex { - public: - explicit GPUIndex(const int &device_id) : gpu_id_(device_id) {} - GPUIndex(const int& device_id, const ResPtr& resource): gpu_id_(device_id), res_(resource){} - - virtual VectorIndexPtr CopyGpuToCpu(const Config &config) = 0; - virtual VectorIndexPtr CopyGpuToGpu(const int64_t &device_id, const Config &config) = 0; - - void SetGpuDevice(const int &gpu_id); - const int64_t &GetGpuDevice(); - - protected: - int64_t gpu_id_; - ResWPtr res_; -}; - -class GPUIVF : public IVF, public GPUIndex { - public: - explicit GPUIVF(const int &device_id) : IVF(), GPUIndex(device_id) {} - explicit GPUIVF(std::shared_ptr index, const int64_t &device_id, ResPtr &resource) - : IVF(std::move(index)), GPUIndex(device_id, resource) {}; - IndexModelPtr Train(const DatasetPtr &dataset, const Config &config) override; - void Add(const DatasetPtr &dataset, const Config &config) override; - void set_index_model(IndexModelPtr model) override; - //DatasetPtr Search(const DatasetPtr &dataset, const Config &config) override; - VectorIndexPtr CopyGpuToCpu(const Config &config) override; - VectorIndexPtr CopyGpuToGpu(const int64_t &device_id, const Config &config) override; - VectorIndexPtr Clone() final; - - // TODO(linxj): Deprecated - virtual IVFIndexPtr Copy_index_gpu_to_cpu(); - - protected: - void search_impl(int64_t n, - const float *data, - int64_t k, - float *distances, - int64_t *labels, - const Config &cfg) override; - BinarySet SerializeImpl() override; - void LoadImpl(const BinarySet &index_binary) override; -}; - -class GPUIVFSQ : public GPUIVF { - public: - explicit GPUIVFSQ(const int &device_id) : GPUIVF(device_id) {} - explicit GPUIVFSQ(std::shared_ptr index, const int64_t &device_id, ResPtr &resource) - : GPUIVF(std::move(index), device_id, resource) {}; - IndexModelPtr Train(const DatasetPtr &dataset, const Config &config) override; - - public: - VectorIndexPtr CopyGpuToCpu(const Config &config) override; -}; - -class GPUIVFPQ : public GPUIVF { - public: - explicit GPUIVFPQ(const int &device_id) : GPUIVF(device_id) {} - IndexModelPtr Train(const DatasetPtr &dataset, const Config &config) override; - - public: - VectorIndexPtr CopyGpuToCpu(const Config &config) override; - - protected: - // TODO(linxj): remove GenParams. - std::shared_ptr GenParams(const Config &config) override; -}; - - -} // namespace knowhere -} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/cloner.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp similarity index 82% rename from cpp/src/core/knowhere/knowhere/index/vector_index/cloner.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp index 4519def30083d55d2a35afb11e2f24ff30b06026..d8ccdd1087bd78a2999b16187c558f7f5f4717a4 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/cloner.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp @@ -16,15 +16,18 @@ // under the License. -#include "knowhere/common/exception.h" -#include "cloner.h" -#include "ivf.h" -#include "gpu_ivf.h" -#include "idmap.h" +#include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/IndexIVF.h" +#include "knowhere/index/vector_index/IndexIVFSQ.h" +#include "knowhere/index/vector_index/IndexIVFPQ.h" +#include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "knowhere/index/vector_index/IndexIDMAP.h" +#include "Cloner.h" namespace zilliz { namespace knowhere { +namespace cloner { VectorIndexPtr CopyGpuToCpu(const VectorIndexPtr &index, const Config &config) { if (auto device_index = std::dynamic_pointer_cast(index)) { @@ -42,7 +45,7 @@ VectorIndexPtr CopyCpuToGpu(const VectorIndexPtr &index, const int64_t &device_i if (auto cpu_index = std::dynamic_pointer_cast(index)) { return cpu_index->CopyCpuToGpu(device_id, config); } else if (auto cpu_index = std::dynamic_pointer_cast(index)) { - KNOWHERE_THROW_MSG("IVFPQ not support tranfer to gpu"); + KNOWHERE_THROW_MSG("IVFPQ not support transfer to gpu"); } else if (auto cpu_index = std::dynamic_pointer_cast(index)) { return cpu_index->CopyCpuToGpu(device_id, config); } else if (auto cpu_index = std::dynamic_pointer_cast(index)) { @@ -52,5 +55,6 @@ VectorIndexPtr CopyCpuToGpu(const VectorIndexPtr &index, const int64_t &device_i } } +} // cloner } } diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/cloner.h b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Cloner.h similarity index 90% rename from cpp/src/core/knowhere/knowhere/index/vector_index/cloner.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Cloner.h index f546ecc7d90febdbb0f080ad5d3fca1b732eef75..a72e4df141ffc2b35a220175b7a66a04f0a9b355 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/cloner.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Cloner.h @@ -18,11 +18,12 @@ #pragma once -#include "vector_index.h" +#include "knowhere/index/vector_index/VectorIndex.h" namespace zilliz { namespace knowhere { +namespace cloner { // TODO(linxj): rename CopyToGpu extern VectorIndexPtr @@ -31,5 +32,6 @@ CopyCpuToGpu(const VectorIndexPtr &index, const int64_t &device_id, const Config extern VectorIndexPtr CopyGpuToCpu(const VectorIndexPtr &index, const Config &config); -} -} \ No newline at end of file +} // cloner +} // knowhere +} // zilliz \ No newline at end of file diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/definitions.h b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Definitions.h similarity index 93% rename from cpp/src/core/knowhere/knowhere/index/vector_index/definitions.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Definitions.h index 3db32e00dc16c2ccd73b0c04a25a4b3499411ab5..fef7b01221f792800b4eba2dc73456cadf70cf6a 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/definitions.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Definitions.h @@ -18,15 +18,15 @@ #pragma once -#include - namespace zilliz { namespace knowhere { +namespace definition { #define META_ROWS ("rows") #define META_DIM ("dimension") #define META_K ("k") -} // namespace knowhere -} // namespace zilliz +} // definition +} // knowhere +} // zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/FaissGpuResourceMgr.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissGpuResourceMgr.cpp similarity index 100% rename from cpp/src/core/knowhere/knowhere/index/vector_index/FaissGpuResourceMgr.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissGpuResourceMgr.cpp diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/FaissGpuResourceMgr.h b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h similarity index 100% rename from cpp/src/core/knowhere/knowhere/index/vector_index/FaissGpuResourceMgr.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissIO.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissIO.cpp new file mode 100644 index 0000000000000000000000000000000000000000..48167fe92577fbb224c76143529adffbab6358b0 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissIO.cpp @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#include + +#include "FaissIO.h" + +namespace zilliz { +namespace knowhere { + +// TODO(linxj): Get From Config File +static size_t magic_num = 2; +size_t MemoryIOWriter::operator()(const void *ptr, size_t size, size_t nitems) { + auto total_need = size * nitems + rp; + + if (!data_) { // data == nullptr + total = total_need * magic_num; + rp = size * nitems; + data_ = new uint8_t[total]; + memcpy((void *) (data_), ptr, rp); + } + + if (total_need > total) { + total = total_need * magic_num; + auto new_data = new uint8_t[total]; + memcpy((void *) new_data, (void *) data_, rp); + delete data_; + data_ = new_data; + + memcpy((void *) (data_ + rp), ptr, size * nitems); + rp = total_need; + } else { + memcpy((void *) (data_ + rp), ptr, size * nitems); + rp = total_need; + } + + return nitems; +} + +size_t MemoryIOReader::operator()(void *ptr, size_t size, size_t nitems) { + if (rp >= total) return 0; + size_t nremain = (total - rp) / size; + if (nremain < nitems) nitems = nremain; + memcpy(ptr, (void *) (data_ + rp), size * nitems); + rp += size * nitems; + return nitems; +} + +} // knowhere +} // zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissIO.h b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissIO.h new file mode 100644 index 0000000000000000000000000000000000000000..cc089ec367f09762a166b4d5fb4e72c7dec80b37 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissIO.h @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include + +namespace zilliz { +namespace knowhere { + +struct MemoryIOWriter : public faiss::IOWriter { + uint8_t *data_ = nullptr; + size_t total = 0; + size_t rp = 0; + + size_t + operator()(const void *ptr, size_t size, size_t nitems) override; +}; + +struct MemoryIOReader : public faiss::IOReader { + uint8_t *data_; + size_t rp = 0; + size_t total = 0; + + size_t + operator()(void *ptr, size_t size, size_t nitems) override; +}; + +} // knowhere +} // zilliz + + + diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.cpp similarity index 93% rename from cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.cpp index b0ed9f6d564b498e1abf3951b6ed6b0b552a418b..a085bfcf48298781fc7e1683df4a8e26cead44a7 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.cpp @@ -18,18 +18,18 @@ #include -#include "kdt_parameters.h" +#include "KDTParameterMgr.h" namespace zilliz { namespace knowhere { const std::vector & -KDTParameterManagement::GetKDTParameters() { +KDTParameterMgr::GetKDTParameters() { return kdt_parameters_; } -KDTParameterManagement::KDTParameterManagement() { +KDTParameterMgr::KDTParameterMgr() { kdt_parameters_ = std::vector{ {"KDTNumber", "1"}, {"NumTopDimensionKDTSplit", "5"}, diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.h b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.h similarity index 80% rename from cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.h index 03f660a0ef8b96dd2bf01c8ba90666a1b0393b70..4b43e642e2877774a356c3802090f6642e4d885e 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.h @@ -27,22 +27,22 @@ namespace knowhere { using KDTParameter = std::pair; -class KDTParameterManagement { +class KDTParameterMgr { public: const std::vector & GetKDTParameters(); public: - static KDTParameterManagement & + static KDTParameterMgr & GetInstance() { - static KDTParameterManagement instance; + static KDTParameterMgr instance; return instance; } - KDTParameterManagement(const KDTParameterManagement &) = delete; - KDTParameterManagement &operator=(const KDTParameterManagement &) = delete; + KDTParameterMgr(const KDTParameterMgr &) = delete; + KDTParameterMgr &operator=(const KDTParameterMgr &) = delete; private: - KDTParameterManagement(); + KDTParameterMgr(); private: std::vector kdt_parameters_; diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/ivf.h b/cpp/src/core/knowhere/knowhere/index/vector_index/ivf.h deleted file mode 100644 index c23b5abcc29f1229252662bcab47fcda1c44c814..0000000000000000000000000000000000000000 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/ivf.h +++ /dev/null @@ -1,157 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - - -#pragma once - -#include -#include - -#include -#include -#include - -#include "vector_index.h" - - -namespace zilliz { -namespace knowhere { - -class BasicIndex { - protected: - explicit BasicIndex(std::shared_ptr index); - virtual BinarySet SerializeImpl(); - virtual void LoadImpl(const BinarySet &index_binary); - virtual void - SealImpl(); - - protected: - std::shared_ptr index_ = nullptr; -}; - -using Graph = std::vector>; - -class IVF : public VectorIndex, protected BasicIndex { - public: - IVF() : BasicIndex(nullptr) {}; - explicit IVF(std::shared_ptr index) : BasicIndex(std::move(index)) {} - VectorIndexPtr Clone() override;; - IndexModelPtr Train(const DatasetPtr &dataset, const Config &config) override; - void set_index_model(IndexModelPtr model) override; - void Add(const DatasetPtr &dataset, const Config &config) override; - void AddWithoutIds(const DatasetPtr &dataset, const Config &config); - DatasetPtr Search(const DatasetPtr &dataset, const Config &config) override; - void GenGraph(const int64_t &k, Graph &graph, const DatasetPtr &dataset, const Config &config); - BinarySet Serialize() override; - void Load(const BinarySet &index_binary) override; - int64_t Count() override; - int64_t Dimension() override; - - void - Seal() override; - - virtual VectorIndexPtr CopyCpuToGpu(const int64_t &device_id, const Config &config); - - - protected: - virtual std::shared_ptr GenParams(const Config &config); - - virtual VectorIndexPtr Clone_impl(const std::shared_ptr &index); - - virtual void search_impl(int64_t n, - const float *data, - int64_t k, - float *distances, - int64_t *labels, - const Config &cfg); - - protected: - std::mutex mutex_; -}; - -using IVFIndexPtr = std::shared_ptr; - -class IVFSQ : public IVF { - public: - explicit IVFSQ(std::shared_ptr index) : IVF(std::move(index)) {} - IVFSQ() = default; - IndexModelPtr Train(const DatasetPtr &dataset, const Config &config) override; - VectorIndexPtr CopyCpuToGpu(const int64_t &device_id, const Config &config) override; - protected: - VectorIndexPtr Clone_impl(const std::shared_ptr &index) override; -}; - -class IVFPQ : public IVF { - public: - explicit IVFPQ(std::shared_ptr index) : IVF(std::move(index)) {} - IVFPQ() = default; - IndexModelPtr Train(const DatasetPtr &dataset, const Config &config) override; - protected: - std::shared_ptr GenParams(const Config &config) override; - VectorIndexPtr Clone_impl(const std::shared_ptr &index) override; -}; - - -//class OPQIVFPQ : public IVFPQ { -// public: -// PreprocessorPtr BuildPreprocessor(const Dataset &dataset, const Config &config) override; -//}; - - -class GPUIVF; - - -struct MemoryIOWriter : public faiss::IOWriter { - uint8_t *data_ = nullptr; - size_t total = 0; - size_t rp = 0; - - size_t operator()(const void *ptr, size_t size, size_t nitems) override; -}; - - -struct MemoryIOReader : public faiss::IOReader { - uint8_t *data_; - size_t rp = 0; - size_t total = 0; - - size_t operator()(void *ptr, size_t size, size_t nitems) override; - -}; - - -class IVFIndexModel : public IndexModel, public BasicIndex { - friend IVF; - friend GPUIVF; - - public: - explicit IVFIndexModel(std::shared_ptr index); - IVFIndexModel() : BasicIndex(nullptr) {}; - BinarySet Serialize() override; - protected: - void SealImpl() override; - public: - void Load(const BinarySet &binary) override; - - protected: - std::mutex mutex_; -}; - -using IVFIndexModelPtr = std::shared_ptr; - -} -} \ No newline at end of file diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp similarity index 99% rename from cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp index d42f4c566241081aa4a9f02ef6e515b783db20f9..c2b5122d655278372ae379baf186b957b8fb4000 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp @@ -22,10 +22,10 @@ #include #include -#include "nsg.h" -#include "knowhere/common/exception.h" -#include "knowhere/common/timer.h" -#include "utils.h" +#include "NSG.h" +#include "knowhere/common/Exception.h" +#include "knowhere/common/Timer.h" +#include "NSGHelper.h" // TODO: enable macro //#include diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg.h b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.h similarity index 99% rename from cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.h index 247d78b97b954c57c12e120182d51906a2c63c09..5291c17280a387f8540c6e752d3b178f90e96c88 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.h @@ -23,7 +23,7 @@ #include #include -#include "neighbor.h" +#include "Neighbor.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/utils.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGHelper.cpp similarity index 99% rename from cpp/src/core/knowhere/knowhere/index/vector_index/nsg/utils.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGHelper.cpp index 96a78bb63f7a7d6284378c5ea64b6b5b8048c1ba..eb87f97f3edcdbfa7a0b6557687e7ca704d2b370 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/utils.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGHelper.cpp @@ -19,7 +19,7 @@ #include #include -#include "utils.h" +#include "NSGHelper.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/utils.h b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGHelper.h similarity index 95% rename from cpp/src/core/knowhere/knowhere/index/vector_index/nsg/utils.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGHelper.h index a57a7738fda3a346184fc9df3bd3c807d9994eb5..f5c13194c403033acd0ac9e0be6917516f41f07c 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/utils.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGHelper.h @@ -23,8 +23,8 @@ #include -#include "nsg.h" -#include "knowhere/common/config.h" +#include "NSG.h" +#include "knowhere/common/Config.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg_io.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGIO.cpp similarity index 99% rename from cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg_io.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGIO.cpp index 34d629ec4b10115690967cb6e455726fe0d2cba7..bcdde8052c351e42c32e72feaed24711e8d0dbee 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg_io.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGIO.cpp @@ -18,7 +18,7 @@ #include -#include "nsg_io.h" +#include "NSGIO.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg_io.h b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGIO.h similarity index 88% rename from cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg_io.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGIO.h index e84d2a98517f18dbc3d4a0783cc9d7289681e576..3d6786c6c236278764a8ed5b418180f038a80275 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg_io.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGIO.h @@ -18,8 +18,9 @@ #pragma once -#include "nsg.h" -#include "knowhere/index/vector_index/ivf.h" +#include "knowhere/index/vector_index/helpers/FaissIO.h" +#include "NSG.h" +#include "knowhere/index/vector_index/IndexIVF.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/neighbor.h b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/Neighbor.h similarity index 100% rename from cpp/src/core/knowhere/knowhere/index/vector_index/nsg/neighbor.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg/Neighbor.h diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/index_io.h b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/index_io.h deleted file mode 100644 index 60e27bd7f66defb91c938e8be21f937d45396b70..0000000000000000000000000000000000000000 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/index_io.h +++ /dev/null @@ -1,32 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - - -#pragma once - -#include "nsg.h" - -namespace zilliz { -namespace knowhere { -namespace algo { - -void read_from_file(NsgIndex* index, const char *filename); -void write_to_file(NsgIndex* index, const char *filename); - -} -} -} diff --git a/cpp/src/core/test/test_idmap.cpp b/cpp/src/core/test/test_idmap.cpp index f85210b574f57719aa6e8c5461006eaf7ec5ac43..82f00455ddd563efec63cace929286c395f738ca 100644 --- a/cpp/src/core/test/test_idmap.cpp +++ b/cpp/src/core/test/test_idmap.cpp @@ -21,10 +21,10 @@ #include #include -#include "knowhere/index/vector_index/idmap.h" -#include "knowhere/adapter/structure.h" -#include "knowhere/index/vector_index/cloner.h" -#include "knowhere/common/exception.h" +#include "knowhere/index/vector_index/IndexIDMAP.h" +#include "knowhere/adapter/Structure.h" +#include "knowhere/index/vector_index/utils/Cloner.h" +#include "knowhere/common/Exception.h" #include "utils.h" diff --git a/cpp/src/core/test/test_ivf.cpp b/cpp/src/core/test/test_ivf.cpp index f1ae3f44a6dccfbed746cca92be91629a86e327c..5d29ae29ac3d6a1209c18d7184949e7f79987601 100644 --- a/cpp/src/core/test/test_ivf.cpp +++ b/cpp/src/core/test/test_ivf.cpp @@ -25,12 +25,12 @@ #include #include -#include "knowhere/index/vector_index/gpu_ivf.h" -#include "knowhere/index/vector_index/ivf.h" -#include "knowhere/adapter/structure.h" -#include "knowhere/index/vector_index/cloner.h" -#include "knowhere/common/exception.h" -#include "knowhere/common/timer.h" +#include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "knowhere/index/vector_index/IndexIVF.h" +#include "knowhere/adapter/Structure.h" +#include "knowhere/index/vector_index/utils/Cloner.h" +#include "knowhere/common/Exception.h" +#include "knowhere/common/Timer.h" #include "utils.h" diff --git a/cpp/src/core/test/test_kdt.cpp b/cpp/src/core/test/test_kdt.cpp index 9581cc8ee0e85655ddf057040d00219d4bfd01a5..8874b14fc907a2bfaf240e20c5ba6d7ae523e8f9 100644 --- a/cpp/src/core/test/test_kdt.cpp +++ b/cpp/src/core/test/test_kdt.cpp @@ -20,12 +20,12 @@ #include #include -#include "knowhere/common/exception.h" +#include "knowhere/common/Exception.h" -#include "knowhere/index/vector_index/cpu_kdt_rng.h" -#include "knowhere/index/vector_index/definitions.h" -#include "knowhere/adapter/sptag.h" -#include "knowhere/adapter/structure.h" +#include "knowhere/index/vector_index/IndexKDT.h" +#include "knowhere/index/vector_index/utils/Definitions.h" +#include "knowhere/adapter/SptagAdapter.h" +#include "knowhere/adapter/Structure.h" #include "utils.h" diff --git a/cpp/src/core/test/utils.h b/cpp/src/core/test/utils.h index df7f11d68952b500894056dfeb7a2ea29335a689..41dbbc1f30118978f1c9d1afd6ab9e25f969c12a 100644 --- a/cpp/src/core/test/utils.h +++ b/cpp/src/core/test/utils.h @@ -23,7 +23,7 @@ #include #include -#include "knowhere/adapter/structure.h" +#include "knowhere/adapter/Structure.h" #include "knowhere/common/Log.h" class DataGen { diff --git a/cpp/src/db/Options.cpp b/cpp/src/db/Options.cpp index 5796b781b2a4fecf525005d88214e811305ac0bf..fd52b851e2d4055836fe01d7fa17bc5e8024edef 100644 --- a/cpp/src/db/Options.cpp +++ b/cpp/src/db/Options.cpp @@ -17,7 +17,7 @@ #include "Options.h" #include "utils/Exception.h" -#include "utils/easylogging++.h" +#include "utils/Log.h" #include #include @@ -56,11 +56,11 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) { std::vector kv; boost::algorithm::split(kv, token, boost::is_any_of(":")); if (kv.size() != 2) { - LOG(WARNING) << "Invalid ArchiveConf Criterias: " << token << " Ignore!"; + ENGINE_LOG_WARNING << "Invalid ArchiveConf Criterias: " << token << " Ignore!"; continue; } if (kv[0] != "disk" && kv[0] != "days") { - LOG(WARNING) << "Invalid ArchiveConf Criterias: " << token << " Ignore!"; + ENGINE_LOG_WARNING << "Invalid ArchiveConf Criterias: " << token << " Ignore!"; continue; } try { @@ -68,20 +68,22 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) { criterias_[kv[0]] = value; } catch (std::out_of_range&){ - LOG(ERROR) << "Out of range: '" << kv[1] << "'"; - throw OutOfRangeException(); + std::string msg = "Out of range: '" + kv[1] + "'"; + ENGINE_LOG_ERROR << msg; + throw InvalidArgumentException(msg); } catch (...){ - LOG(ERROR) << "Invalid argument: '" << kv[1] << "'"; - throw InvalidArgumentException(); + std::string msg = "Invalid argument: '" + kv[1] + "'"; + ENGINE_LOG_ERROR << msg; + throw InvalidArgumentException(msg); } } } void ArchiveConf::ParseType(const std::string& type) { if (type != "delete" && type != "swap") { - LOG(ERROR) << "Invalid argument: type='" << type << "'"; - throw InvalidArgumentException(); + std::string msg = "Invalid argument: type='" + type + "'"; + throw InvalidArgumentException(msg); } type_ = type; } diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index 2fb868f548cc94ec1d51a9f4917f2061d6272244..7b9bac2e3a27b2118cf141646d88f0a7070c9085 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -25,7 +25,7 @@ #include "src/wrapper/vec_index.h" #include "src/wrapper/vec_impl.h" -#include "knowhere/common/exception.h" +#include "knowhere/common/Exception.h" #include @@ -45,14 +45,18 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, nlist_(nlist) { index_ = CreatetVecIndex(EngineType::FAISS_IDMAP); - if (!index_) throw Exception("Create Empty VecIndex"); + if (!index_) { + throw Exception(DB_ERROR, "Could not create VecIndex"); + } Config build_cfg; build_cfg["dim"] = dimension; build_cfg["metric_type"] = (metric_type_ == MetricType::IP) ? "IP" : "L2"; AutoGenParams(index_->GetType(), 0, build_cfg); auto ec = std::static_pointer_cast(index_)->Build(build_cfg); - if (ec != KNOWHERE_SUCCESS) { throw Exception("Build index error"); } + if (ec != KNOWHERE_SUCCESS) { + throw Exception(DB_ERROR, "Build index error"); + } } ExecutionEngineImpl::ExecutionEngineImpl(VecIndexPtr index, @@ -273,7 +277,7 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t auto to_index = CreatetVecIndex(engine_type); if (!to_index) { - throw Exception("Create Empty VecIndex"); + throw Exception(DB_ERROR, "Could not create VecIndex"); } Config build_cfg; @@ -287,7 +291,7 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t from_index->GetRawVectors(), from_index->GetRawIds(), build_cfg); - if (ec != KNOWHERE_SUCCESS) { throw Exception("Build index error"); } + if (ec != KNOWHERE_SUCCESS) { throw Exception(DB_ERROR, "Build index error"); } return std::make_shared(to_index, location, engine_type, metric_type_, nlist_); } diff --git a/cpp/src/db/meta/Meta.h b/cpp/src/db/meta/Meta.h index ee4972ae6be73c53bff381239eed90c1fa2e4ff7..58168e5fee3cabb8eba089bfdedf40714b967f8c 100644 --- a/cpp/src/db/meta/Meta.h +++ b/cpp/src/db/meta/Meta.h @@ -31,6 +31,9 @@ namespace milvus { namespace engine { namespace meta { +static const char* META_TABLES = "Tables"; +static const char* META_TABLEFILES = "TableFiles"; + class Meta { public: virtual ~Meta() = default; diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index a07d4c82cf951303a159b08d1febc3dd853e35bd..282df32eb18f9a2c18b714bce28348c80cce34c7 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -153,8 +153,9 @@ Status MySQLMetaImpl::Initialize() { } Query InitializeQuery = connectionPtr->query(); - InitializeQuery << "CREATE TABLE IF NOT EXISTS Tables (" << - "id BIGINT PRIMARY KEY AUTO_INCREMENT, " << + InitializeQuery << "CREATE TABLE IF NOT EXISTS " << + META_TABLES << " " << + "(id BIGINT PRIMARY KEY AUTO_INCREMENT, " << "table_id VARCHAR(255) UNIQUE NOT NULL, " << "state INT NOT NULL, " << "dimension SMALLINT NOT NULL, " << @@ -171,8 +172,9 @@ Status MySQLMetaImpl::Initialize() { return HandleException("Initialization Error", InitializeQuery.error()); } - InitializeQuery << "CREATE TABLE IF NOT EXISTS TableFiles (" << - "id BIGINT PRIMARY KEY AUTO_INCREMENT, " << + InitializeQuery << "CREATE TABLE IF NOT EXISTS " << + META_TABLEFILES << " " << + "(id BIGINT PRIMARY KEY AUTO_INCREMENT, " << "table_id VARCHAR(255) NOT NULL, " << "engine_type INT DEFAULT 1 NOT NULL, " << "file_id VARCHAR(255) NOT NULL, " << @@ -233,7 +235,8 @@ Status MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id, Query dropPartitionsByDatesQuery = connectionPtr->query(); - dropPartitionsByDatesQuery << "UPDATE TableFiles " << + dropPartitionsByDatesQuery << "UPDATE " << + META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << "," << "updated_time = " << utils::GetMicroSecTimeStamp() << " " << "WHERE table_id = " << quote << table_id << " AND " << @@ -266,7 +269,8 @@ Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) { if (table_schema.table_id_.empty()) { NextTableId(table_schema.table_id_); } else { - createTableQuery << "SELECT state FROM Tables " << + createTableQuery << "SELECT state FROM " << + META_TABLES << " " << "WHERE table_id = " << quote << table_schema.table_id_ << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); @@ -297,8 +301,9 @@ Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) { std::string nlist = std::to_string(table_schema.nlist_); std::string metric_type = std::to_string(table_schema.metric_type_); - createTableQuery << "INSERT INTO Tables VALUES" << - "(" << id << ", " << quote << table_id << ", " << state << ", " << dimension << ", " << + createTableQuery << "INSERT INTO " << + META_TABLES << " " << + "VALUES(" << id << ", " << quote << table_id << ", " << state << ", " << dimension << ", " << created_on << ", " << flag << ", " << index_file_size << ", " << engine_type << ", " << nlist << ", " << metric_type << ");"; @@ -348,7 +353,8 @@ Status MySQLMetaImpl::FilesByType(const std::string &table_id, Query hasNonIndexFilesQuery = connectionPtr->query(); //since table_id is a unique column we just need to check whether it exists or not - hasNonIndexFilesQuery << "SELECT file_id, file_type FROM TableFiles " << + hasNonIndexFilesQuery << "SELECT file_id, file_type FROM " << + META_TABLEFILES << " " << "WHERE table_id = " << quote << table_id << " AND " << "file_type in (" << types << ");"; @@ -418,8 +424,8 @@ Status MySQLMetaImpl::UpdateTableIndex(const std::string &table_id, const TableI } Query updateTableIndexParamQuery = connectionPtr->query(); - updateTableIndexParamQuery << "SELECT id, state, dimension, created_on " << - "FROM Tables " << + updateTableIndexParamQuery << "SELECT id, state, dimension, created_on FROM " << + META_TABLES << " " << "WHERE table_id = " << quote << table_id << " AND " << "state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; @@ -435,7 +441,8 @@ Status MySQLMetaImpl::UpdateTableIndex(const std::string &table_id, const TableI uint16_t dimension = resRow["dimension"]; int64_t created_on = resRow["created_on"]; - updateTableIndexParamQuery << "UPDATE Tables " << + updateTableIndexParamQuery << "UPDATE " << + META_TABLES << " " << "SET id = " << id << ", " << "state = " << state << ", " << "dimension = " << dimension << ", " << @@ -476,7 +483,8 @@ Status MySQLMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) } Query updateTableFlagQuery = connectionPtr->query(); - updateTableFlagQuery << "UPDATE Tables " << + updateTableFlagQuery << "UPDATE " << + META_TABLES << " " << "SET flag = " << flag << " " << "WHERE table_id = " << quote << table_id << ";"; @@ -507,8 +515,8 @@ Status MySQLMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex } Query describeTableIndexQuery = connectionPtr->query(); - describeTableIndexQuery << "SELECT engine_type, nlist, index_file_size, metric_type " << - "FROM Tables " << + describeTableIndexQuery << "SELECT engine_type, nlist, index_file_size, metric_type FROM " << + META_TABLES << " " << "WHERE table_id = " << quote << table_id << " AND " << "state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; @@ -549,7 +557,8 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { Query dropTableIndexQuery = connectionPtr->query(); //soft delete index files - dropTableIndexQuery << "UPDATE TableFiles " << + dropTableIndexQuery << "UPDATE " << + META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << "," << "updated_time = " << utils::GetMicroSecTimeStamp() << " " << "WHERE table_id = " << quote << table_id << " AND " << @@ -562,7 +571,8 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { } //set all backup file to raw - dropTableIndexQuery << "UPDATE TableFiles " << + dropTableIndexQuery << "UPDATE " << + META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::RAW) << "," << "updated_time = " << utils::GetMicroSecTimeStamp() << " " << "WHERE table_id = " << quote << table_id << " AND " << @@ -575,7 +585,8 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { } //set table index type to raw - dropTableIndexQuery << "UPDATE Tables " << + dropTableIndexQuery << "UPDATE " << + META_TABLES << " " << "SET engine_type = " << std::to_string(DEFAULT_ENGINE_TYPE) << "," << "nlist = " << std::to_string(DEFAULT_NLIST) << ", " << "metric_type = " << std::to_string(DEFAULT_METRIC_TYPE) << " " << @@ -609,7 +620,8 @@ Status MySQLMetaImpl::DeleteTable(const std::string &table_id) { //soft delete table Query deleteTableQuery = connectionPtr->query(); // - deleteTableQuery << "UPDATE Tables " << + deleteTableQuery << "UPDATE " << + META_TABLES << " " << "SET state = " << std::to_string(TableSchema::TO_DELETE) << " " << "WHERE table_id = " << quote << table_id << ";"; @@ -645,7 +657,8 @@ Status MySQLMetaImpl::DeleteTableFiles(const std::string &table_id) { //soft delete table files Query deleteTableFilesQuery = connectionPtr->query(); // - deleteTableFilesQuery << "UPDATE TableFiles " << + deleteTableFilesQuery << "UPDATE " << + META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " << "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << "WHERE table_id = " << quote << table_id << " AND " << @@ -676,9 +689,8 @@ Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { } Query describeTableQuery = connectionPtr->query(); - describeTableQuery << "SELECT id, state, dimension, created_on, " << - "flag, index_file_size, engine_type, nlist, metric_type " << - "FROM Tables " << + describeTableQuery << "SELECT id, state, dimension, created_on, flag, index_file_size, engine_type, nlist, metric_type FROM " << + META_TABLES << " " << "WHERE table_id = " << quote << table_schema.table_id_ << " " << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; @@ -732,7 +744,8 @@ Status MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { Query hasTableQuery = connectionPtr->query(); //since table_id is a unique column we just need to check whether it exists or not hasTableQuery << "SELECT EXISTS " << - "(SELECT 1 FROM Tables " << + "(SELECT 1 FROM " << + META_TABLES << " " << "WHERE table_id = " << quote << table_id << " " << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << "AS " << quote << "check" << ";"; @@ -764,8 +777,8 @@ Status MySQLMetaImpl::AllTables(std::vector &table_schema_array) { } Query allTablesQuery = connectionPtr->query(); - allTablesQuery << "SELECT id, table_id, dimension, engine_type, nlist, index_file_size, metric_type " << - "FROM Tables " << + allTablesQuery << "SELECT id, table_id, dimension, engine_type, nlist, index_file_size, metric_type FROM " << + META_TABLES << " " << "WHERE state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::AllTables: " << allTablesQuery.str(); @@ -846,8 +859,9 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { Query createTableFileQuery = connectionPtr->query(); - createTableFileQuery << "INSERT INTO TableFiles VALUES" << - "(" << id << ", " << quote << table_id << ", " << engine_type << ", " << + createTableFileQuery << "INSERT INTO " << + META_TABLEFILES << " " << + "VALUES(" << id << ", " << quote << table_id << ", " << engine_type << ", " << quote << file_id << ", " << file_type << ", " << file_size << ", " << row_count << ", " << updated_time << ", " << created_on << ", " << date << ");"; @@ -883,8 +897,8 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { } Query filesToIndexQuery = connectionPtr->query(); - filesToIndexQuery << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date, created_on " << - "FROM TableFiles " << + filesToIndexQuery << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date, created_on FROM " << + META_TABLEFILES << " " << "WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToIndex: " << filesToIndexQuery.str(); @@ -967,8 +981,8 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, } Query filesToSearchQuery = connectionPtr->query(); - filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date " << - "FROM TableFiles " << + filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date FROM " << + META_TABLEFILES << " " << "WHERE table_id = " << quote << table_id; if (!partition.empty()) { @@ -1086,8 +1100,8 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, } Query filesToMergeQuery = connectionPtr->query(); - filesToMergeQuery << "SELECT id, table_id, file_id, file_type, file_size, row_count, date, engine_type, created_on " << - "FROM TableFiles " << + filesToMergeQuery << "SELECT id, table_id, file_id, file_type, file_size, row_count, date, engine_type, created_on FROM " << + META_TABLEFILES << " " << "WHERE table_id = " << quote << table_id << " AND " << "file_type = " << std::to_string(TableFileSchema::RAW) << " " << "ORDER BY row_count DESC" << ";"; @@ -1177,8 +1191,8 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id, } Query getTableFileQuery = connectionPtr->query(); - getTableFileQuery << "SELECT id, engine_type, file_id, file_type, file_size, row_count, date, created_on " << - "FROM TableFiles " << + getTableFileQuery << "SELECT id, engine_type, file_id, file_type, file_size, row_count, date, created_on FROM " << + META_TABLEFILES << " " << "WHERE table_id = " << quote << table_id << " AND " << "(" << idStr << ") AND " << "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; @@ -1259,7 +1273,8 @@ Status MySQLMetaImpl::Archive() { } Query archiveQuery = connectionPtr->query(); - archiveQuery << "UPDATE TableFiles " << + archiveQuery << "UPDATE " << + META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " << "WHERE created_on < " << std::to_string(now - usecs) << " AND " << "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; @@ -1299,8 +1314,8 @@ Status MySQLMetaImpl::Size(uint64_t &result) { } Query getSizeQuery = connectionPtr->query(); - getSizeQuery << "SELECT IFNULL(SUM(file_size),0) AS sum " << - "FROM TableFiles " << + getSizeQuery << "SELECT IFNULL(SUM(file_size),0) AS sum FROM " << + META_TABLEFILES << " " << "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::Size: " << getSizeQuery.str(); @@ -1339,8 +1354,8 @@ Status MySQLMetaImpl::DiscardFiles(long long to_discard_size) { } Query discardFilesQuery = connectionPtr->query(); - discardFilesQuery << "SELECT id, file_size " << - "FROM TableFiles " << + discardFilesQuery << "SELECT id, file_size FROM " << + META_TABLEFILES << " " << "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << " " << "ORDER BY id ASC " << "LIMIT 10;"; @@ -1369,7 +1384,8 @@ Status MySQLMetaImpl::DiscardFiles(long long to_discard_size) { std::string idsToDiscardStr = idsToDiscardSS.str(); idsToDiscardStr = idsToDiscardStr.substr(0, idsToDiscardStr.size() - 4); //remove the last " OR " - discardFilesQuery << "UPDATE TableFiles " << + discardFilesQuery << "UPDATE " << + META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " << "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << "WHERE " << idsToDiscardStr << ";"; @@ -1406,7 +1422,8 @@ Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { //if the table has been deleted, just mark the table file as TO_DELETE //clean thread will delete the file later - updateTableFileQuery << "SELECT state FROM Tables " << + updateTableFileQuery << "SELECT state FROM " << + META_TABLES << " " << "WHERE table_id = " << quote << file_schema.table_id_ << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); @@ -1433,7 +1450,8 @@ Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { std::string created_on = std::to_string(file_schema.created_on_); std::string date = std::to_string(file_schema.date_); - updateTableFileQuery << "UPDATE TableFiles " << + updateTableFileQuery << "UPDATE " << + META_TABLEFILES << " " << "SET table_id = " << quote << table_id << ", " << "engine_type = " << engine_type << ", " << "file_id = " << quote << file_id << ", " << @@ -1470,7 +1488,8 @@ Status MySQLMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { Query updateTableFilesToIndexQuery = connectionPtr->query(); - updateTableFilesToIndexQuery << "UPDATE TableFiles " << + updateTableFilesToIndexQuery << "UPDATE " << + META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " " << "WHERE table_id = " << quote << table_id << " AND " << "file_type = " << std::to_string(TableFileSchema::RAW) << ";"; @@ -1508,7 +1527,8 @@ Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { } updateTableFilesQuery << "SELECT EXISTS " << - "(SELECT 1 FROM Tables " << + "(SELECT 1 FROM " << + META_TABLES << " " << "WHERE table_id = " << quote << file_schema.table_id_ << " " << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << "AS " << quote << "check" << ";"; @@ -1539,7 +1559,8 @@ Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { std::string created_on = std::to_string(file_schema.created_on_); std::string date = std::to_string(file_schema.date_); - updateTableFilesQuery << "UPDATE TableFiles " << + updateTableFilesQuery << "UPDATE " << + META_TABLEFILES << " " << "SET table_id = " << quote << table_id << ", " << "engine_type = " << engine_type << ", " << "file_id = " << quote << file_id << ", " << @@ -1582,8 +1603,8 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { } Query cleanUpFilesWithTTLQuery = connectionPtr->query(); - cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, date " << - "FROM TableFiles " << + cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, date FROM " << + META_TABLEFILES << " " << "WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " AND " << "updated_time < " << std::to_string(now - seconds * US_PS) << ";"; @@ -1626,8 +1647,9 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { std::string idsToDeleteStr = idsToDeleteSS.str(); idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " - cleanUpFilesWithTTLQuery << "DELETE FROM TableFiles WHERE " << - idsToDeleteStr << ";"; + cleanUpFilesWithTTLQuery << "DELETE FROM " << + META_TABLEFILES << " " << + "WHERE " << idsToDeleteStr << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); @@ -1653,8 +1675,8 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { } Query cleanUpFilesWithTTLQuery = connectionPtr->query(); - cleanUpFilesWithTTLQuery << "SELECT id, table_id " << - "FROM Tables " << + cleanUpFilesWithTTLQuery << "SELECT id, table_id FROM " << + META_TABLES << " " << "WHERE state = " << std::to_string(TableSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); @@ -1675,8 +1697,9 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { } std::string idsToDeleteStr = idsToDeleteSS.str(); idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " - cleanUpFilesWithTTLQuery << "DELETE FROM Tables WHERE " << - idsToDeleteStr << ";"; + cleanUpFilesWithTTLQuery << "DELETE FROM " << + META_TABLES << " " << + "WHERE " << idsToDeleteStr << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); @@ -1704,8 +1727,8 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { for(auto& table_id : table_ids) { Query cleanUpFilesWithTTLQuery = connectionPtr->query(); - cleanUpFilesWithTTLQuery << "SELECT file_id " << - "FROM TableFiles " << + cleanUpFilesWithTTLQuery << "SELECT file_id FROM " << + META_TABLEFILES << " " << "WHERE table_id = " << quote << table_id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); @@ -1736,7 +1759,7 @@ Status MySQLMetaImpl::CleanUp() { cleanUpQuery << "SELECT table_name " << "FROM information_schema.tables " << "WHERE table_schema = " << quote << mysql_connection_pool_->getDB() << " " << - "AND table_name = " << quote << "TableFiles" << ";"; + "AND table_name = " << quote << META_TABLEFILES << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); @@ -1744,7 +1767,7 @@ Status MySQLMetaImpl::CleanUp() { if (!res.empty()) { ENGINE_LOG_DEBUG << "Remove table file type as NEW"; - cleanUpQuery << "DELETE FROM TableFiles WHERE file_type IN (" + cleanUpQuery << "DELETE FROM " << META_TABLEFILES << " WHERE file_type IN (" << std::to_string(TableFileSchema::NEW) << "," << std::to_string(TableFileSchema::NEW_MERGE) << "," << std::to_string(TableFileSchema::NEW_INDEX) << ");"; @@ -1785,8 +1808,8 @@ Status MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { Query countQuery = connectionPtr->query(); - countQuery << "SELECT row_count " << - "FROM TableFiles " << + countQuery << "SELECT row_count FROM " << + META_TABLEFILES << " " << "WHERE table_id = " << quote << table_id << " AND " << "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << @@ -1820,7 +1843,7 @@ Status MySQLMetaImpl::DropAll() { } Query dropTableQuery = connectionPtr->query(); - dropTableQuery << "DROP TABLE IF EXISTS Tables, TableFiles;"; + dropTableQuery << "DROP TABLE IF EXISTS " << META_TABLES << ", " << META_TABLEFILES << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropAll: " << dropTableQuery.str(); diff --git a/cpp/src/db/meta/SqliteMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp index 564a8eae4d0bd4f3dadca6f6fbac7ee0a7de43eb..a28038d019e2de4bb853b3f3a965b9738fc7f290 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -19,6 +19,7 @@ #include "db/IDGenerator.h" #include "db/Utils.h" #include "utils/Log.h" +#include "utils/Exception.h" #include "MetaConsts.h" #include "metrics/Metrics.h" @@ -55,7 +56,7 @@ Status HandleException(const std::string &desc, const char* what = nullptr) { inline auto StoragePrototype(const std::string &path) { return make_storage(path, - make_table("Tables", + make_table(META_TABLES, make_column("id", &TableSchema::id_, primary_key()), make_column("table_id", &TableSchema::table_id_, unique()), make_column("state", &TableSchema::state_), @@ -66,7 +67,7 @@ inline auto StoragePrototype(const std::string &path) { make_column("engine_type", &TableSchema::engine_type_), make_column("nlist", &TableSchema::nlist_), make_column("metric_type", &TableSchema::metric_type_)), - make_table("TableFiles", + make_table(META_TABLEFILES, make_column("id", &TableFileSchema::id_, primary_key()), make_column("table_id", &TableFileSchema::table_id_), make_column("engine_type", &TableFileSchema::engine_type_), @@ -122,6 +123,17 @@ Status SqliteMetaImpl::Initialize() { ConnectorPtr = std::make_unique(StoragePrototype(options_.path + "/meta.sqlite")); + //old meta could be recreated since schema changed, throw exception if meta schema is not compatible + auto ret = ConnectorPtr->sync_schema_simulate(); + if(ret.find(META_TABLES) != ret.end() + && sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLES]) { + throw Exception(DB_INCOMPATIB_META, "Meta schema is created by Milvus old version"); + } + if(ret.find(META_TABLEFILES) != ret.end() + && sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLEFILES]) { + throw Exception(DB_INCOMPATIB_META, "Meta schema is created by Milvus old version"); + } + ConnectorPtr->sync_schema(); ConnectorPtr->open_forever(); // thread safe option ConnectorPtr->pragma.journal_mode(journal_mode::WAL); // WAL => write ahead log @@ -1246,8 +1258,8 @@ Status SqliteMetaImpl::DropAll() { ENGINE_LOG_DEBUG << "Drop all sqlite meta"; try { - ConnectorPtr->drop_table("Tables"); - ConnectorPtr->drop_table("TableFiles"); + ConnectorPtr->drop_table(META_TABLES); + ConnectorPtr->drop_table(META_TABLEFILES); } catch (std::exception &e) { return HandleException("Encounter exception when drop all meta", e.what()); } diff --git a/cpp/src/main.cpp b/cpp/src/main.cpp index 54f9541effacd050f037393b1de02a3ea2966daf..5a61c5bfb4508d2d1ef2f560ff7e32f00a431c17 100644 --- a/cpp/src/main.cpp +++ b/cpp/src/main.cpp @@ -102,10 +102,6 @@ main(int argc, char *argv[]) { } } - server::Server &server = server::Server::Instance(); - server.Init(start_daemonized, pid_filename, config_filename, log_config_file); - server.Start(); - /* Handle Signal */ signal(SIGHUP, server::SignalUtil::HandleSignal); signal(SIGINT, server::SignalUtil::HandleSignal); @@ -114,6 +110,10 @@ main(int argc, char *argv[]) { signal(SIGUSR2, server::SignalUtil::HandleSignal); signal(SIGTERM, server::SignalUtil::HandleSignal); + server::Server &server = server::Server::Instance(); + server.Init(start_daemonized, pid_filename, config_filename, log_config_file); + server.Start(); + /* wait signal */ pause(); diff --git a/cpp/src/scheduler/SchedInst.cpp b/cpp/src/scheduler/SchedInst.cpp index 424bc9f778db71baa678da9b478ab82fe3af7241..3b14158e58d6fbe2002a65c549883716b6243f1a 100644 --- a/cpp/src/scheduler/SchedInst.cpp +++ b/cpp/src/scheduler/SchedInst.cpp @@ -19,7 +19,7 @@ #include "SchedInst.h" #include "server/ServerConfig.h" #include "ResourceFactory.h" -#include "knowhere/index/vector_index/gpu_ivf.h" +#include "knowhere/index/vector_index/IndexGPUIVF.h" #include "Utils.h" diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 115a636621c94b04376783712dd308a273b1610f..1409fff597ae18cc4967581714c2a61c8a878db0 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -109,15 +109,10 @@ Status DBWrapper::StartService() { } //create db instance - std::string msg = opt.meta.path; try { db_ = engine::DBFactory::Build(opt); } catch(std::exception& ex) { - msg = ex.what(); - } - - if(db_ == nullptr) { - std::cerr << "ERROR! Failed to open database: " << msg << std::endl; + std::cerr << "ERROR! Failed to open database: " << ex.what() << std::endl; kill(0, SIGUSR1); } diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index 38b41a5331461386b7acd678f85720077489dcdd..2d0a4e5c99f7488959102a87a4fbef438aaef818 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -196,8 +196,8 @@ Server::Start() { server::Metrics::GetInstance().Init(); server::SystemInfo::GetInstance().Init(); - std::cout << "Milvus server start successfully." << std::endl; StartService(); + std::cout << "Milvus server start successfully." << std::endl; } catch (std::exception &ex) { std::cerr << "Milvus server encounter exception: " << ex.what(); diff --git a/cpp/src/utils/Error.h b/cpp/src/utils/Error.h index d7008115f972b5f75814332a5e53965610f78ddf..672aa9284887d6226c8d1b01de64641c997532ae 100644 --- a/cpp/src/utils/Error.h +++ b/cpp/src/utils/Error.h @@ -86,6 +86,7 @@ constexpr ErrorCode DB_ERROR = ToDbErrorCode(2); constexpr ErrorCode DB_NOT_FOUND = ToDbErrorCode(3); constexpr ErrorCode DB_ALREADY_EXIST = ToDbErrorCode(4); constexpr ErrorCode DB_INVALID_PATH = ToDbErrorCode(5); +constexpr ErrorCode DB_INCOMPATIB_META = ToDbErrorCode(6); //knowhere error code constexpr ErrorCode KNOWHERE_ERROR = ToKnowhereErrorCode(1); diff --git a/cpp/src/utils/Exception.h b/cpp/src/utils/Exception.h index acb796c61ded06fbf368a46641b8e80ad0833ae3..6542f913af2f550d1bf452cacb5fe2cc09136360 100644 --- a/cpp/src/utils/Exception.h +++ b/cpp/src/utils/Exception.h @@ -17,6 +17,8 @@ #pragma once +#include "utils/Error.h" + #include #include @@ -25,13 +27,14 @@ namespace milvus { class Exception : public std::exception { public: - Exception(const std::string& message) - : message_(message) { + Exception(ErrorCode code, const std::string& message) + : code_(code_), + message_(message) { } - Exception() - : message_() { - } + ErrorCode code() const throw() { + return code_; + } virtual const char* what() const throw() { if (message_.empty()) { @@ -44,20 +47,21 @@ public: virtual ~Exception() throw() {}; protected: - + ErrorCode code_; std::string message_; }; class InvalidArgumentException : public Exception { public: - InvalidArgumentException() : Exception("Invalid Argument"){}; - InvalidArgumentException(const std::string& message) : Exception(message) {}; -}; + InvalidArgumentException() + : Exception(SERVER_INVALID_ARGUMENT, "Invalid Argument") { + + }; + InvalidArgumentException(const std::string& message) + : Exception(SERVER_INVALID_ARGUMENT, message) { + + }; -class OutOfRangeException : public Exception { -public: - OutOfRangeException() : Exception("Out Of Range"){}; - OutOfRangeException(const std::string& message) : Exception(message) {}; }; } // namespace milvus diff --git a/cpp/src/utils/Log.h b/cpp/src/utils/Log.h index 5bb28696e21facde872fbfe30c647bf23f1cd0f8..2610a3a8bf904d59fa8ebae71197fbe2fc20664c 100644 --- a/cpp/src/utils/Log.h +++ b/cpp/src/utils/Log.h @@ -24,7 +24,6 @@ namespace milvus { ///////////////////////////////////////////////////////////////////////////////////////////////// #define SERVER_DOMAIN_NAME "[SERVER] " -#define SERVER_ERROR_TEXT "SERVER Error:" #define SERVER_LOG_TRACE LOG(TRACE) << SERVER_DOMAIN_NAME #define SERVER_LOG_DEBUG LOG(DEBUG) << SERVER_DOMAIN_NAME @@ -35,7 +34,6 @@ namespace milvus { ///////////////////////////////////////////////////////////////////////////////////////////////// #define ENGINE_DOMAIN_NAME "[ENGINE] " -#define ENGINE_ERROR_TEXT "ENGINE Error:" #define ENGINE_LOG_TRACE LOG(TRACE) << ENGINE_DOMAIN_NAME #define ENGINE_LOG_DEBUG LOG(DEBUG) << ENGINE_DOMAIN_NAME @@ -46,7 +44,6 @@ namespace milvus { ///////////////////////////////////////////////////////////////////////////////////////////////// #define WRAPPER_DOMAIN_NAME "[WRAPPER] " -#define WRAPPER_ERROR_TEXT "WRAPPER Error:" #define WRAPPER_LOG_TRACE LOG(TRACE) << WRAPPER_DOMAIN_NAME #define WRAPPER_LOG_DEBUG LOG(DEBUG) << WRAPPER_DOMAIN_NAME diff --git a/cpp/src/utils/LogUtil.cpp b/cpp/src/utils/LogUtil.cpp index e4af1104db3ccae24dd89f1236a2e3a674917733..cfdbbdc768e9ab56cdb45ffc2fb0180e8d93f5d0 100644 --- a/cpp/src/utils/LogUtil.cpp +++ b/cpp/src/utils/LogUtil.cpp @@ -17,10 +17,8 @@ #include "LogUtil.h" #include "server/ServerConfig.h" -#include "easylogging++.h" #include - #include #include diff --git a/cpp/src/wrapper/KnowhereResource.cpp b/cpp/src/wrapper/KnowhereResource.cpp index c8c03423a81864cb5d49ed26cd88b15d3cf97ca7..007858de37ec43d9e44804d7af33d130972d4858 100644 --- a/cpp/src/wrapper/KnowhereResource.cpp +++ b/cpp/src/wrapper/KnowhereResource.cpp @@ -17,7 +17,7 @@ #include "KnowhereResource.h" -#include "knowhere/index/vector_index/FaissGpuResourceMgr.h" +#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" #include "server/ServerConfig.h" #include diff --git a/cpp/src/wrapper/data_transfer.h b/cpp/src/wrapper/data_transfer.h index 7f41af98db1d7e5746555fc49c04a1e7bafc4f16..10aaf355c91aebcffa3d1e4820a5fd49df90c6a1 100644 --- a/cpp/src/wrapper/data_transfer.h +++ b/cpp/src/wrapper/data_transfer.h @@ -18,7 +18,7 @@ #pragma once -#include "knowhere/adapter/structure.h" +#include "knowhere/adapter/Structure.h" namespace zilliz { diff --git a/cpp/src/wrapper/vec_impl.cpp b/cpp/src/wrapper/vec_impl.cpp index b6c15b72fb662d13edf954337ac729a72422bb20..accb612c2d39717ef84f311c3dc58f0af4ca96c4 100644 --- a/cpp/src/wrapper/vec_impl.cpp +++ b/cpp/src/wrapper/vec_impl.cpp @@ -17,10 +17,10 @@ #include "utils/Log.h" -#include "knowhere/index/vector_index/idmap.h" -#include "knowhere/index/vector_index/gpu_ivf.h" -#include "knowhere/common/exception.h" -#include "knowhere/index/vector_index/cloner.h" +#include "knowhere/index/vector_index/IndexIDMAP.h" +#include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/helpers/Cloner.h" #include "vec_impl.h" #include "data_transfer.h" @@ -153,7 +153,7 @@ IndexType VecIndexImpl::GetType() { VecIndexPtr VecIndexImpl::CopyToGpu(const int64_t &device_id, const Config &cfg) { // TODO(linxj): exception handle - auto gpu_index = zilliz::knowhere::CopyCpuToGpu(index_, device_id, cfg); + auto gpu_index = zilliz::knowhere::cloner::CopyCpuToGpu(index_, device_id, cfg); auto new_index = std::make_shared(gpu_index, ConvertToGpuIndexType(type)); new_index->dim = dim; return new_index; @@ -161,7 +161,7 @@ VecIndexPtr VecIndexImpl::CopyToGpu(const int64_t &device_id, const Config &cfg) VecIndexPtr VecIndexImpl::CopyToCpu(const Config &cfg) { // TODO(linxj): exception handle - auto cpu_index = zilliz::knowhere::CopyGpuToCpu(index_, cfg); + auto cpu_index = zilliz::knowhere::cloner::CopyGpuToCpu(index_, cfg); auto new_index = std::make_shared(cpu_index, ConvertToCpuIndexType(type)); new_index->dim = dim; return new_index; diff --git a/cpp/src/wrapper/vec_impl.h b/cpp/src/wrapper/vec_impl.h index b10dc6b711acbc9c686e293cf23b9baa231d3f18..5e5edbd1991c27f0118d88326a264e800cfb4d79 100644 --- a/cpp/src/wrapper/vec_impl.h +++ b/cpp/src/wrapper/vec_impl.h @@ -18,7 +18,7 @@ #pragma once -#include "knowhere/index/vector_index/vector_index.h" +#include "knowhere/index/vector_index/VectorIndex.h" #include "vec_index.h" diff --git a/cpp/src/wrapper/vec_index.cpp b/cpp/src/wrapper/vec_index.cpp index b1c377d032b69cf97ae41bc35ffe62debe6ccbf2..7ed176d96e96c0d76cb0dc2f0b570fe93b1d2e5d 100644 --- a/cpp/src/wrapper/vec_index.cpp +++ b/cpp/src/wrapper/vec_index.cpp @@ -15,12 +15,16 @@ // specific language governing permissions and limitations // under the License. -#include "knowhere/index/vector_index/ivf.h" -#include "knowhere/index/vector_index/idmap.h" -#include "knowhere/index/vector_index/gpu_ivf.h" -#include "knowhere/index/vector_index/cpu_kdt_rng.h" -#include "knowhere/index/vector_index/nsg_index.h" -#include "knowhere/common/exception.h" +#include "knowhere/index/vector_index/IndexIVF.h" +#include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "knowhere/index/vector_index/IndexIVFSQ.h" +#include "knowhere/index/vector_index/IndexGPUIVFSQ.h" +#include "knowhere/index/vector_index/IndexIVFPQ.h" +#include "knowhere/index/vector_index/IndexGPUIVFPQ.h" +#include "knowhere/index/vector_index/IndexIDMAP.h" +#include "knowhere/index/vector_index/IndexKDT.h" +#include "knowhere/index/vector_index/IndexNSG.h" +#include "knowhere/common/Exception.h" #include "vec_index.h" #include "vec_impl.h" diff --git a/cpp/src/wrapper/vec_index.h b/cpp/src/wrapper/vec_index.h index 3ffa0eeec8de220bdcd90672fedd5ff63dcd64d1..331fd2833b7446593806b22a725aa7195f16aefb 100644 --- a/cpp/src/wrapper/vec_index.h +++ b/cpp/src/wrapper/vec_index.h @@ -23,8 +23,8 @@ #include "utils/Error.h" -#include "knowhere/common/config.h" -#include "knowhere/common/binary_set.h" +#include "knowhere/common/Config.h" +#include "knowhere/common/BinarySet.h" namespace zilliz { diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index a1fd14a8f410e3a671674b34c1d69ab7b0f0a257..f2f81215388723850b21946babcdbcedbe6e69a3 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -24,10 +24,7 @@ #include "utils/CommonUtil.h" #include -#include "utils/easylogging++.h" - #include - #include #include diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 62271b9a3b51180b9c38d9fb32ad93eeafc5edbf..7a6ee31243b009bacd5f898524f33ecc4d27d896 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -15,17 +15,16 @@ // specific language governing permissions and limitations // under the License. -#include -#include -#include "utils/easylogging++.h" -#include -#include - #include "utils.h" #include "db/meta/SqliteMetaImpl.h" #include "db/Utils.h" #include "db/meta/MetaConsts.h" +#include +#include +#include +#include + using namespace zilliz::milvus; using namespace zilliz::milvus::engine; diff --git a/cpp/unittest/db/misc_test.cpp b/cpp/unittest/db/misc_test.cpp index ff530b0a734c5a172aaee1bd9f0e3ed1913a816c..dd388dab7ab78cf1a155ef6d041f11446438d01d 100644 --- a/cpp/unittest/db/misc_test.cpp +++ b/cpp/unittest/db/misc_test.cpp @@ -21,7 +21,6 @@ #include "db/Utils.h" #include "utils/Status.h" #include "utils/Exception.h" -#include "utils/easylogging++.h" #include #include @@ -31,13 +30,13 @@ using namespace zilliz::milvus; TEST(DBMiscTest, EXCEPTION_TEST) { - Exception ex1(""); + Exception ex1(100, "error"); std::string what = ex1.what(); - ASSERT_FALSE(what.empty()); + ASSERT_EQ(what, "error"); + ASSERT_EQ(ex1.code(), 100); - OutOfRangeException ex2; - what = ex2.what(); - ASSERT_FALSE(what.empty()); + InvalidArgumentException ex2; + ASSERT_EQ(ex2.code(), SERVER_INVALID_ARGUMENT); } TEST(DBMiscTest, OPTIONS_TEST) { diff --git a/cpp/unittest/db/mysql_db_test.cpp b/cpp/unittest/db/mysql_db_test.cpp index 9d99b4095becd4528ab7f2a19e6a0114eaf77c8a..53ce51ca45f5434a7eabf39206bafa8fe5bf64f1 100644 --- a/cpp/unittest/db/mysql_db_test.cpp +++ b/cpp/unittest/db/mysql_db_test.cpp @@ -21,7 +21,6 @@ #include "db/meta/MetaConsts.h" #include -#include "utils/easylogging++.h" #include #include diff --git a/cpp/unittest/db/mysql_meta_test.cpp b/cpp/unittest/db/mysql_meta_test.cpp index e99565886d3cedcf1694f4337f1a4aa64021f311..a7c9837f146292c95c3785211cd61a1d8cd5df52 100644 --- a/cpp/unittest/db/mysql_meta_test.cpp +++ b/cpp/unittest/db/mysql_meta_test.cpp @@ -15,20 +15,17 @@ // specific language governing permissions and limitations // under the License. -#include -#include -#include "utils/easylogging++.h" -#include -#include - #include "utils.h" #include "db/meta/MySQLMetaImpl.h" #include "db/Utils.h" #include "db/meta/MetaConsts.h" -#include "mysql++/mysql++.h" - #include +#include +#include +#include +#include +#include using namespace zilliz::milvus; using namespace zilliz::milvus::engine; diff --git a/cpp/unittest/db/utils.cpp b/cpp/unittest/db/utils.cpp index 03ea11bf085dc54b11b25e53215118a108a9d2fa..e9950a8574f221d5fd410837b8b6863e0c06bc61 100644 --- a/cpp/unittest/db/utils.cpp +++ b/cpp/unittest/db/utils.cpp @@ -25,8 +25,7 @@ #include "cache/CpuCacheMgr.h" #include "db/DBFactory.h" #include "db/Options.h" -#include "server/ServerConfig.h" -#include "knowhere/index/vector_index/gpu_ivf.h" +#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" INITIALIZE_EASYLOGGINGPP diff --git a/cpp/unittest/main.cpp b/cpp/unittest/main.cpp index b0c2be6d5ae02c34ddd9191781c2af53161e566f..33a860945dfa035354d2e329cd76b7089c669aa5 100644 --- a/cpp/unittest/main.cpp +++ b/cpp/unittest/main.cpp @@ -15,13 +15,13 @@ // specific language governing permissions and limitations // under the License. -#include -#include #include "utils/easylogging++.h" - #include "server/ServerConfig.h" #include "utils/CommonUtil.h" +#include +#include + INITIALIZE_EASYLOGGINGPP using namespace zilliz::milvus; diff --git a/cpp/unittest/server/util_test.cpp b/cpp/unittest/server/util_test.cpp index b582c9a17afb74e145189a348141629b54e6fd99..169b2e23ebaad8b91c0f2312800d45c4fb02c6f1 100644 --- a/cpp/unittest/server/util_test.cpp +++ b/cpp/unittest/server/util_test.cpp @@ -15,14 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include -#include -#include "utils/easylogging++.h" -#include -#include -#include -#include - +#include "utils/SignalUtil.h" #include "utils/CommonUtil.h" #include "utils/Error.h" #include "utils/StringHelpFunctions.h" @@ -32,6 +25,12 @@ #include "utils/ValidationUtil.h" #include "db/engine/ExecutionEngine.h" +#include +#include +#include +#include +#include + using namespace zilliz::milvus; namespace { diff --git a/cpp/unittest/wrapper/wrapper_test.cpp b/cpp/unittest/wrapper/wrapper_test.cpp index ac185efbb3f8137462a9b15917a1313a149874ed..b052758b6ccfa76dec4e3b626f7a9178cddc9b04 100644 --- a/cpp/unittest/wrapper/wrapper_test.cpp +++ b/cpp/unittest/wrapper/wrapper_test.cpp @@ -15,15 +15,13 @@ // specific language governing permissions and limitations // under the License. - -#include #include "utils/easylogging++.h" - #include "src/wrapper/vec_index.h" -#include "knowhere/index/vector_index/gpu_ivf.h" - +#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" #include "utils.h" +#include + INITIALIZE_EASYLOGGINGPP using namespace zilliz::milvus::engine;