diff --git a/cpp/src/core/CMakeLists.txt b/cpp/src/core/CMakeLists.txt index b18af23f1e10bb5503fb26335f57cc43a55d109e..bef3f5e2f685be12023f87fa3b80d07121364e59 100644 --- a/cpp/src/core/CMakeLists.txt +++ b/cpp/src/core/CMakeLists.txt @@ -84,7 +84,7 @@ include(DefineOptionsCore) include(BuildUtilsCore) include(ThirdPartyPackagesCore) -add_subdirectory(src) +add_subdirectory(knowhere) if (BUILD_COVERAGE STREQUAL "ON") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage") diff --git a/cpp/src/core/include/knowhere/adapter/arrow.h b/cpp/src/core/include/knowhere/adapter/arrow.h deleted file mode 100644 index 338ab691df05d04179f1e556f171fc9fe72c2688..0000000000000000000000000000000000000000 --- a/cpp/src/core/include/knowhere/adapter/arrow.h +++ /dev/null @@ -1,18 +0,0 @@ - -#pragma once - -#include -#include - - -namespace zilliz { -namespace knowhere { - -ArrayPtr -CopyArray(const ArrayPtr &origin); - -SchemaPtr -CopySchema(const SchemaPtr &origin); - -} // namespace knowhere -} // namespace zilliz diff --git a/cpp/src/core/include/knowhere/adapter/sptag.h b/cpp/src/core/include/knowhere/adapter/sptag.h deleted file mode 100644 index 08c4a625eec6f569e557594221104cef0620bf1b..0000000000000000000000000000000000000000 --- a/cpp/src/core/include/knowhere/adapter/sptag.h +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once - -#include -#include -#include - - -namespace zilliz { -namespace knowhere { - -std::shared_ptr -ConvertToVectorSet(const DatasetPtr &dataset); - -std::shared_ptr -ConvertToMetadataSet(const DatasetPtr &dataset); - -std::vector -ConvertToQueryResult(const DatasetPtr &dataset, const Config &config); - -DatasetPtr -ConvertToDataset(std::vector query_results); - -} // namespace knowhere -} // namespace zilliz diff --git a/cpp/src/core/include/knowhere/common/config.h b/cpp/src/core/include/knowhere/common/config.h deleted file mode 100644 index 3a3fee1219e2f205a00308a9ef96771d05ce3dc4..0000000000000000000000000000000000000000 --- a/cpp/src/core/include/knowhere/common/config.h +++ /dev/null @@ -1,14 +0,0 @@ -#pragma once - -#include - - -namespace zilliz { -namespace knowhere { - - -using Config = jsoncons::json; - - -} // namespace knowhere -} // namespace zilliz diff --git a/cpp/src/core/include/knowhere/common/device_type.h b/cpp/src/core/include/knowhere/common/device_type.h deleted file mode 100644 index 09fa3626437a2091e686d7b3777616c069e4e6fe..0000000000000000000000000000000000000000 --- a/cpp/src/core/include/knowhere/common/device_type.h +++ /dev/null @@ -1,10 +0,0 @@ -#pragma once - - -namespace zilliz { -namespace sched { -namespace master { - -} // namespace master -} // namespace sched -} // namespace zilliz diff --git a/cpp/src/core/include/knowhere/common/error.h b/cpp/src/core/include/knowhere/common/error.h deleted file mode 100644 index ba7b5cf315840b8025e43bc317666c5e58abccf2..0000000000000000000000000000000000000000 --- a/cpp/src/core/include/knowhere/common/error.h +++ /dev/null @@ -1,40 +0,0 @@ -#pragma once - -#include -#include "zlibrary/error/error.h" - - -namespace zilliz { -namespace knowhere { - -using Error = zilliz::lib::ErrorCode; - -constexpr Error STORE_SUCCESS = zilliz::lib::SUCCESS_CODE; - -constexpr Error ERROR_CODE_BASE = 0x36000; -constexpr Error ERROR_CODE_END = 0x37000; - -constexpr Error -ToGlobalErrorCode(const Error error_code) { - return zilliz::lib::ToGlobalErrorCode(error_code, ERROR_CODE_BASE); -} - -class Exception : public zilliz::lib::Exception { - public: - Exception(const Error error_code, - const std::string &message = nullptr) - : zilliz::lib::Exception(error_code, "KNOWHERE", message) {} -}; - -constexpr Error UNEXPECTED = ToGlobalErrorCode(0x001); -constexpr Error UNSUPPORTED = ToGlobalErrorCode(0x002); -constexpr Error NULL_POINTER = ToGlobalErrorCode(0x003); -constexpr Error OVERFLOW = ToGlobalErrorCode(0x004); -constexpr Error INVALID_ARGUMENT = ToGlobalErrorCode(0x005); -constexpr Error UNSUPPORTED_TYPE = ToGlobalErrorCode(0x006); - - -} // namespace store -} // namespace zilliz - -using Error = zilliz::store::Error; diff --git a/cpp/src/core/include/knowhere/common/id.h b/cpp/src/core/include/knowhere/common/id.h deleted file mode 100644 index dc823e6e22a3d95da89404ce6d0a9463d98cbbc8..0000000000000000000000000000000000000000 --- a/cpp/src/core/include/knowhere/common/id.h +++ /dev/null @@ -1,42 +0,0 @@ -#pragma once - -//#include "zcommon/id/id.h" -//using ID = zilliz::common::ID; - -#include -#include - -namespace zilliz { -namespace knowhere { - - - -class ID { - public: - constexpr static int64_t kIDSize = 20; - - public: - const int32_t * - data() const { return content_; } - - int32_t * - mutable_data() { return content_; } - - bool - IsValid() const; - - std::string - ToString() const; - - bool - operator==(const ID &that) const; - - bool - operator<(const ID &that) const; - - protected: - int32_t content_[5] = {}; -}; - -} // namespace knowhere -} // namespace zilliz diff --git a/cpp/src/core/include/knowhere/common/schema.h b/cpp/src/core/include/knowhere/common/schema.h deleted file mode 100644 index fe877214705e7ba1ac9f78d977af2525fa2f75b1..0000000000000000000000000000000000000000 --- a/cpp/src/core/include/knowhere/common/schema.h +++ /dev/null @@ -1,21 +0,0 @@ -#pragma once - -#include -#include "arrow/type.h" - - -namespace zilliz { -namespace knowhere { - - -using DataType = arrow::DataType; -using Field = arrow::Field; -using FieldPtr = std::shared_ptr; -using Schema = arrow::Schema; -using SchemaPtr = std::shared_ptr; -using SchemaConstPtr = std::shared_ptr; - - - -} // namespace knowhere -} // namespace zilliz diff --git a/cpp/src/core/include/knowhere/common/tensor.h b/cpp/src/core/include/knowhere/common/tensor.h deleted file mode 100644 index 93b7642cd1fd379cdad493cb6674c52e8ada5646..0000000000000000000000000000000000000000 --- a/cpp/src/core/include/knowhere/common/tensor.h +++ /dev/null @@ -1,16 +0,0 @@ -#pragma once - -#include -#include "arrow/tensor.h" - - -namespace zilliz { -namespace knowhere { - - -using Tensor = arrow::Tensor; -using TensorPtr = std::shared_ptr; - - -} // namespace knowhere -} // namespace zilliz diff --git a/cpp/src/core/include/knowhere/index/index.h b/cpp/src/core/include/knowhere/index/index.h deleted file mode 100644 index 40f3a8d5c04d3f489056bbb2f78c24b67b937551..0000000000000000000000000000000000000000 --- a/cpp/src/core/include/knowhere/index/index.h +++ /dev/null @@ -1,49 +0,0 @@ -#pragma once - -#include -#include "knowhere/common/binary_set.h" -#include "knowhere/common/dataset.h" -#include "knowhere/index/index_type.h" -#include "knowhere/index/index_model.h" -#include "knowhere/index/preprocessor/preprocessor.h" - - -namespace zilliz { -namespace knowhere { - - -class Index { - public: - virtual BinarySet - Serialize() = 0; - - virtual void - Load(const BinarySet &index_binary) = 0; - - // @throw - virtual DatasetPtr - Search(const DatasetPtr &dataset, const Config &config) = 0; - - public: - IndexType - idx_type() const { return idx_type_; } - - void - set_idx_type(IndexType idx_type) { idx_type_ = idx_type; } - - virtual void - set_preprocessor(PreprocessorPtr preprocessor) {} - - virtual void - set_index_model(IndexModelPtr model) {} - - private: - IndexType idx_type_; -}; - - -using IndexPtr = std::shared_ptr; - - -} // namespace knowhere -} // namespace zilliz diff --git a/cpp/src/core/include/knowhere/index/index_model.h b/cpp/src/core/include/knowhere/index/index_model.h deleted file mode 100644 index 4b3a4e439c9e3b9e283d3868e7a0bc0b51213c05..0000000000000000000000000000000000000000 --- a/cpp/src/core/include/knowhere/index/index_model.h +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once - -#include -#include "knowhere/common/binary_set.h" - -namespace zilliz { -namespace knowhere { - - -class IndexModel { - public: - virtual BinarySet - Serialize() = 0; - - virtual void - Load(const BinarySet &binary) = 0; -}; - -using IndexModelPtr = std::shared_ptr; - - - -} // namespace knowhere -} // namespace zilliz diff --git a/cpp/src/core/include/knowhere/index/index_type.h b/cpp/src/core/include/knowhere/index/index_type.h deleted file mode 100644 index c7f9e39057f22eba77a5a282220dec3ceaad2bde..0000000000000000000000000000000000000000 --- a/cpp/src/core/include/knowhere/index/index_type.h +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once - - -namespace zilliz { -namespace knowhere { - - -enum class IndexType { - kUnknown = 0, - kVecIdxBegin = 100, - kVecIVFFlat = kVecIdxBegin, - kVecIdxEnd, -}; - - -} // namespace knowhere -} // namespace zilliz diff --git a/cpp/src/core/include/knowhere/index/preprocessor/preprocessor.h b/cpp/src/core/include/knowhere/index/preprocessor/preprocessor.h deleted file mode 100644 index 368276e51374d6c605ddf666a195cc45c7b8b625..0000000000000000000000000000000000000000 --- a/cpp/src/core/include/knowhere/index/preprocessor/preprocessor.h +++ /dev/null @@ -1,22 +0,0 @@ -#pragma once - -#include -#include "knowhere/common/dataset.h" - - -namespace zilliz { -namespace knowhere { - - -class Preprocessor { - public: - virtual DatasetPtr - Preprocess(const DatasetPtr &input) = 0; -}; - - -using PreprocessorPtr = std::shared_ptr; - - -} // namespace knowhere -} // namespace zilliz diff --git a/cpp/src/core/include/knowhere/index/vector_index/definitions.h b/cpp/src/core/include/knowhere/index/vector_index/definitions.h deleted file mode 100644 index 990459758454327768f588875edfef36d81b858c..0000000000000000000000000000000000000000 --- a/cpp/src/core/include/knowhere/index/vector_index/definitions.h +++ /dev/null @@ -1,14 +0,0 @@ -#pragma once - -#include - - -namespace zilliz { -namespace knowhere { - -#define META_ROWS ("rows") -#define META_DIM ("dimension") -#define META_K ("k") - -} // namespace knowhere -} // namespace zilliz diff --git a/cpp/src/core/include/knowhere/index/vector_index/kdt_parameters.h b/cpp/src/core/include/knowhere/index/vector_index/kdt_parameters.h deleted file mode 100644 index 6f2d631d42ae336b0fa492ef11c5ae19aca75d64..0000000000000000000000000000000000000000 --- a/cpp/src/core/include/knowhere/index/vector_index/kdt_parameters.h +++ /dev/null @@ -1,34 +0,0 @@ -#pragma once - -#include -#include - - -namespace zilliz { -namespace knowhere { - -using KDTParameter = std::pair; - -class KDTParameterManagement { - public: - const std::vector & - GetKDTParameters(); - - public: - static KDTParameterManagement & - GetInstance() { - static KDTParameterManagement instance; - return instance; - } - - KDTParameterManagement(const KDTParameterManagement &) = delete; - KDTParameterManagement &operator=(const KDTParameterManagement &) = delete; - private: - KDTParameterManagement(); - - private: - std::vector kdt_parameters_; -}; - -} // namespace knowhere -} // namespace zilliz diff --git a/cpp/src/core/include/knowhere/index/vector_index/nsg/nsg.i b/cpp/src/core/include/knowhere/index/vector_index/nsg/nsg.i deleted file mode 100644 index d66dc490299465e1dd3ae070a083f1e5e6026a7c..0000000000000000000000000000000000000000 --- a/cpp/src/core/include/knowhere/index/vector_index/nsg/nsg.i +++ /dev/null @@ -1,15 +0,0 @@ -%module nsg -%{ -#define SWIG_FILE_WITH_INIT -#include - -/* Include the header in the wrapper code */ -#include "nsg.h" - - -%} - - -/* Parse the header file */ -%include "index.h" - diff --git a/cpp/src/core/include/knowhere/index/vector_index/vector_index.h b/cpp/src/core/include/knowhere/index/vector_index/vector_index.h deleted file mode 100644 index 1618bc01bb7cb6441ede7313f9d86b5b0bc7230c..0000000000000000000000000000000000000000 --- a/cpp/src/core/include/knowhere/index/vector_index/vector_index.h +++ /dev/null @@ -1,45 +0,0 @@ -#pragma once - - -#include -#include "knowhere/common/config.h" -#include "knowhere/common/dataset.h" -#include "knowhere/index/index.h" -#include "knowhere/index/preprocessor/preprocessor.h" - - -namespace zilliz { -namespace knowhere { - -class VectorIndex; -using VectorIndexPtr = std::shared_ptr; - - -class VectorIndex : public Index { - public: - virtual PreprocessorPtr - BuildPreprocessor(const DatasetPtr &dataset, const Config &config) { return nullptr; } - - virtual IndexModelPtr - Train(const DatasetPtr &dataset, const Config &config) { return nullptr; } - - virtual void - Add(const DatasetPtr &dataset, const Config &config) = 0; - - virtual void - Seal() = 0; - - virtual VectorIndexPtr - Clone() = 0; - - virtual int64_t - Count() = 0; - - virtual int64_t - Dimension() = 0; -}; - - - -} // namespace knowhere -} // namespace zilliz diff --git a/cpp/src/core/src/CMakeLists.txt b/cpp/src/core/knowhere/CMakeLists.txt similarity index 97% rename from cpp/src/core/src/CMakeLists.txt rename to cpp/src/core/knowhere/CMakeLists.txt index a51a62b44ad48c2887e29e13965a0f7d8d1f5829..6e6d81c02ca1c983eb60b5197858764fa4cfdb62 100644 --- a/cpp/src/core/src/CMakeLists.txt +++ b/cpp/src/core/knowhere/CMakeLists.txt @@ -5,7 +5,7 @@ include_directories(${TBB_DIR}/include) include_directories(${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) link_directories(${CUDA_TOOLKIT_ROOT_DIR}/lib64) -include_directories(${CORE_SOURCE_DIR}/include) +include_directories(${CORE_SOURCE_DIR}/knowhere) include_directories(${CORE_SOURCE_DIR}/thirdparty) include_directories(${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService) include_directories(${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include) @@ -107,7 +107,7 @@ INSTALL(FILES ${CORE_SOURCE_DIR}/thirdparty/tbb/libtbb.so ) set(CORE_INCLUDE_DIRS - ${CORE_SOURCE_DIR}/include + ${CORE_SOURCE_DIR}/knowhere ${CORE_SOURCE_DIR}/thirdparty ${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include diff --git a/cpp/src/core/src/knowhere/adapter/arrow.cpp b/cpp/src/core/knowhere/knowhere/adapter/arrow.cpp similarity index 58% rename from cpp/src/core/src/knowhere/adapter/arrow.cpp rename to cpp/src/core/knowhere/knowhere/adapter/arrow.cpp index c70c584278f1fb4247935bac760584c3288a1eaa..c4804d33220c353bf4d18cc3ce9870cdaa10030c 100644 --- a/cpp/src/core/src/knowhere/adapter/arrow.cpp +++ b/cpp/src/core/knowhere/knowhere/adapter/arrow.cpp @@ -1,5 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. -#include "knowhere/adapter/arrow.h" + +#include "arrow.h" namespace zilliz { namespace knowhere { diff --git a/cpp/src/core/knowhere/knowhere/adapter/arrow.h b/cpp/src/core/knowhere/knowhere/adapter/arrow.h new file mode 100644 index 0000000000000000000000000000000000000000..318336e703ba5491bab78ab809e913c377754acd --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/adapter/arrow.h @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include + +#include "knowhere/common/array.h" + + +namespace zilliz { +namespace knowhere { + +ArrayPtr +CopyArray(const ArrayPtr &origin); + +SchemaPtr +CopySchema(const SchemaPtr &origin); + +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/include/knowhere/adapter/faiss_adopt.h b/cpp/src/core/knowhere/knowhere/adapter/faiss_adopt.h similarity index 100% rename from cpp/src/core/include/knowhere/adapter/faiss_adopt.h rename to cpp/src/core/knowhere/knowhere/adapter/faiss_adopt.h diff --git a/cpp/src/core/src/knowhere/adapter/sptag.cpp b/cpp/src/core/knowhere/knowhere/adapter/sptag.cpp similarity index 83% rename from cpp/src/core/src/knowhere/adapter/sptag.cpp rename to cpp/src/core/knowhere/knowhere/adapter/sptag.cpp index a724a27d91e9ba0c9a81a52bbf2478d27cc8e0ca..995d068577328339b9df72acf182ddce10966969 100644 --- a/cpp/src/core/src/knowhere/adapter/sptag.cpp +++ b/cpp/src/core/knowhere/knowhere/adapter/sptag.cpp @@ -1,7 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. #include "knowhere/index/vector_index/definitions.h" -#include "knowhere/adapter/sptag.h" -#include "knowhere/adapter/structure.h" +#include "sptag.h" +#include "structure.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/adapter/sptag.h b/cpp/src/core/knowhere/knowhere/adapter/sptag.h new file mode 100644 index 0000000000000000000000000000000000000000..eb7886c26319f6e12cf81d78529f262d2e6f0ed6 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/adapter/sptag.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include + +#include + +#include "knowhere/common/dataset.h" + +namespace zilliz { +namespace knowhere { + +std::shared_ptr +ConvertToVectorSet(const DatasetPtr &dataset); + +std::shared_ptr +ConvertToMetadataSet(const DatasetPtr &dataset); + +std::vector +ConvertToQueryResult(const DatasetPtr &dataset, const Config &config); + +DatasetPtr +ConvertToDataset(std::vector query_results); + +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/src/knowhere/adapter/structure.cpp b/cpp/src/core/knowhere/knowhere/adapter/structure.cpp similarity index 98% rename from cpp/src/core/src/knowhere/adapter/structure.cpp rename to cpp/src/core/knowhere/knowhere/adapter/structure.cpp index f30479dd5aa0cc995e42c75d950822b0c305bb4f..10b6af8138b06cec19f3b38e16c9336feee67822 100644 --- a/cpp/src/core/src/knowhere/adapter/structure.cpp +++ b/cpp/src/core/knowhere/knowhere/adapter/structure.cpp @@ -16,7 +16,7 @@ // under the License. -#include "knowhere/adapter/structure.h" +#include "structure.h" namespace zilliz { diff --git a/cpp/src/core/include/knowhere/adapter/structure.h b/cpp/src/core/knowhere/knowhere/adapter/structure.h similarity index 97% rename from cpp/src/core/include/knowhere/adapter/structure.h rename to cpp/src/core/knowhere/knowhere/adapter/structure.h index 9ffac4fdac59395e6fb3153500d4a340070a8187..76a03a23e4c83ba3dfec916e3b282bed867d6e58 100644 --- a/cpp/src/core/include/knowhere/adapter/structure.h +++ b/cpp/src/core/knowhere/knowhere/adapter/structure.h @@ -19,7 +19,7 @@ #pragma once #include -#include +#include "knowhere/common/dataset.h" namespace zilliz { diff --git a/cpp/src/core/include/knowhere/common/array.h b/cpp/src/core/knowhere/knowhere/common/array.h similarity index 52% rename from cpp/src/core/include/knowhere/common/array.h rename to cpp/src/core/knowhere/knowhere/common/array.h index f3d0310596f1cf2de4c2996aa74e991f16bc9184..1860524e8a2c1384ec6121ac5aaef53db021c037 100644 --- a/cpp/src/core/include/knowhere/common/array.h +++ b/cpp/src/core/knowhere/knowhere/common/array.h @@ -1,7 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #pragma once #include "arrow/array.h" -#include "knowhere/common/schema.h" +#include "schema.h" namespace zilliz { diff --git a/cpp/src/core/include/knowhere/common/binary_set.h b/cpp/src/core/knowhere/knowhere/common/binary_set.h similarity index 58% rename from cpp/src/core/include/knowhere/common/binary_set.h rename to cpp/src/core/knowhere/knowhere/common/binary_set.h index b83d96dfe42131fc7afae11b9acfecfaedc229dd..019c42a2b951ab58b24f660e3ab449c03afc9c9c 100644 --- a/cpp/src/core/include/knowhere/common/binary_set.h +++ b/cpp/src/core/knowhere/knowhere/common/binary_set.h @@ -1,3 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #pragma once #include @@ -5,7 +23,7 @@ #include #include -#include "knowhere/common/id.h" +#include "id.h" namespace zilliz { diff --git a/cpp/src/core/include/knowhere/common/buffer.h b/cpp/src/core/knowhere/knowhere/common/buffer.h similarity index 56% rename from cpp/src/core/include/knowhere/common/buffer.h rename to cpp/src/core/knowhere/knowhere/common/buffer.h index 44867e760ac4ebc09c204833f6a8ea3d832f460b..019c0487b707842b1fbdfbfbe39f475e32f920a1 100644 --- a/cpp/src/core/include/knowhere/common/buffer.h +++ b/cpp/src/core/knowhere/knowhere/common/buffer.h @@ -1,6 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #pragma once #include + #include "arrow/buffer.h" diff --git a/cpp/src/core/knowhere/knowhere/common/config.h b/cpp/src/core/knowhere/knowhere/common/config.h new file mode 100644 index 0000000000000000000000000000000000000000..399555c0e92b332bf104238f7e754e01f5633c4b --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/common/config.h @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include + + +namespace zilliz { +namespace knowhere { + + +using Config = jsoncons::json; + + +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/include/knowhere/common/dataset.h b/cpp/src/core/knowhere/knowhere/common/dataset.h similarity index 77% rename from cpp/src/core/include/knowhere/common/dataset.h rename to cpp/src/core/knowhere/knowhere/common/dataset.h index ee1d6e5100d4cfaad9d47d58e63f874f5208e717..068bbe435658b9358d467caec9f0fe9b8c94c642 100644 --- a/cpp/src/core/include/knowhere/common/dataset.h +++ b/cpp/src/core/knowhere/knowhere/common/dataset.h @@ -1,12 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #pragma once #include #include -#include "knowhere/common/array.h" -#include "knowhere/common/buffer.h" -#include "knowhere/common/tensor.h" -#include "knowhere/common/schema.h" -#include "knowhere/common/config.h" + +#include "array.h" +#include "buffer.h" +#include "tensor.h" +#include "schema.h" +#include "config.h" #include "knowhere/adapter/arrow.h" diff --git a/cpp/src/core/src/knowhere/common/exception.cpp b/cpp/src/core/knowhere/knowhere/common/exception.cpp similarity index 97% rename from cpp/src/core/src/knowhere/common/exception.cpp rename to cpp/src/core/knowhere/knowhere/common/exception.cpp index f46cbfc35ad585ac4f5bbe42c42cfe30121541dc..fcf3886d9b52e8f7b3c1d8bb86033763323e0066 100644 --- a/cpp/src/core/src/knowhere/common/exception.cpp +++ b/cpp/src/core/knowhere/knowhere/common/exception.cpp @@ -16,9 +16,10 @@ // under the License. -#include "knowhere/common/exception.h" #include +#include "exception.h" + namespace zilliz { namespace knowhere { diff --git a/cpp/src/core/include/knowhere/common/exception.h b/cpp/src/core/knowhere/knowhere/common/exception.h similarity index 100% rename from cpp/src/core/include/knowhere/common/exception.h rename to cpp/src/core/knowhere/knowhere/common/exception.h diff --git a/cpp/src/core/knowhere/knowhere/common/id.h b/cpp/src/core/knowhere/knowhere/common/id.h new file mode 100644 index 0000000000000000000000000000000000000000..87e1467c4487a8f03cc40cdeb8fb288777996e29 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/common/id.h @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +//#include "zcommon/id/id.h" +//using ID = zilliz::common::ID; + +#include +#include + +namespace zilliz { +namespace knowhere { + + + +class ID { + public: + constexpr static int64_t kIDSize = 20; + + public: + const int32_t * + data() const { return content_; } + + int32_t * + mutable_data() { return content_; } + + bool + IsValid() const; + + std::string + ToString() const; + + bool + operator==(const ID &that) const; + + bool + operator<(const ID &that) const; + + protected: + int32_t content_[5] = {}; +}; + +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/common/schema.h b/cpp/src/core/knowhere/knowhere/common/schema.h new file mode 100644 index 0000000000000000000000000000000000000000..1ab3d42f8c85b096e4f68a2b3c9b0bb3692bb26a --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/common/schema.h @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include + +#include "arrow/type.h" + + +namespace zilliz { +namespace knowhere { + + +using DataType = arrow::DataType; +using Field = arrow::Field; +using FieldPtr = std::shared_ptr; +using Schema = arrow::Schema; +using SchemaPtr = std::shared_ptr; +using SchemaConstPtr = std::shared_ptr; + + + +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/common/tensor.h b/cpp/src/core/knowhere/knowhere/common/tensor.h new file mode 100644 index 0000000000000000000000000000000000000000..8b92f1f891b66830b54132625a412c75ab344742 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/common/tensor.h @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include + +#include "arrow/tensor.h" + + +namespace zilliz { +namespace knowhere { + + +using Tensor = arrow::Tensor; +using TensorPtr = std::shared_ptr; + + +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/src/knowhere/common/timer.cpp b/cpp/src/core/knowhere/knowhere/common/timer.cpp similarity index 98% rename from cpp/src/core/src/knowhere/common/timer.cpp rename to cpp/src/core/knowhere/knowhere/common/timer.cpp index 08654413ff986fab914d3ce9c8018d2c02da0371..f57de535d426022398fca3b9537c1d28e4dc535d 100644 --- a/cpp/src/core/src/knowhere/common/timer.cpp +++ b/cpp/src/core/knowhere/knowhere/common/timer.cpp @@ -18,7 +18,7 @@ #include // TODO(linxj): using Log instead -#include "knowhere/common/timer.h" +#include "timer.h" namespace zilliz { namespace knowhere { diff --git a/cpp/src/core/include/knowhere/common/timer.h b/cpp/src/core/knowhere/knowhere/common/timer.h similarity index 100% rename from cpp/src/core/include/knowhere/common/timer.h rename to cpp/src/core/knowhere/knowhere/common/timer.h diff --git a/cpp/src/core/knowhere/knowhere/index/index.h b/cpp/src/core/knowhere/knowhere/index/index.h new file mode 100644 index 0000000000000000000000000000000000000000..275a5fd3873ab579ddddbb9792e6c94b8edfdd7a --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/index.h @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include + +#include "knowhere/common/binary_set.h" +#include "knowhere/common/dataset.h" +#include "index_type.h" +#include "index_model.h" +#include "knowhere/index/preprocessor/preprocessor.h" + + +namespace zilliz { +namespace knowhere { + + +class Index { + public: + virtual BinarySet + Serialize() = 0; + + virtual void + Load(const BinarySet &index_binary) = 0; + + // @throw + virtual DatasetPtr + Search(const DatasetPtr &dataset, const Config &config) = 0; + + public: + IndexType + idx_type() const { return idx_type_; } + + void + set_idx_type(IndexType idx_type) { idx_type_ = idx_type; } + + virtual void + set_preprocessor(PreprocessorPtr preprocessor) {} + + virtual void + set_index_model(IndexModelPtr model) {} + + private: + IndexType idx_type_; +}; + + +using IndexPtr = std::shared_ptr; + + +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/index_model.h b/cpp/src/core/knowhere/knowhere/index/index_model.h new file mode 100644 index 0000000000000000000000000000000000000000..d00a629d7e7f4ed55bfb8853ff487af23a2a9776 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/index_model.h @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include +#include "knowhere/common/binary_set.h" + +namespace zilliz { +namespace knowhere { + + +class IndexModel { + public: + virtual BinarySet + Serialize() = 0; + + virtual void + Load(const BinarySet &binary) = 0; +}; + +using IndexModelPtr = std::shared_ptr; + + + +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/index_type.h b/cpp/src/core/knowhere/knowhere/index/index_type.h new file mode 100644 index 0000000000000000000000000000000000000000..3ece6287f5584a0a466e767b9ce028a91d6a0be8 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/index_type.h @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + + +namespace zilliz { +namespace knowhere { + + +enum class IndexType { + kUnknown = 0, + kVecIdxBegin = 100, + kVecIVFFlat = kVecIdxBegin, + kVecIdxEnd, +}; + + +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/src/knowhere/index/preprocessor/normalize.cpp b/cpp/src/core/knowhere/knowhere/index/preprocessor/normalize.cpp similarity index 100% rename from cpp/src/core/src/knowhere/index/preprocessor/normalize.cpp rename to cpp/src/core/knowhere/knowhere/index/preprocessor/normalize.cpp diff --git a/cpp/src/core/include/knowhere/index/preprocessor/normalize.h b/cpp/src/core/knowhere/knowhere/index/preprocessor/normalize.h similarity index 100% rename from cpp/src/core/include/knowhere/index/preprocessor/normalize.h rename to cpp/src/core/knowhere/knowhere/index/preprocessor/normalize.h diff --git a/cpp/src/core/knowhere/knowhere/index/preprocessor/preprocessor.h b/cpp/src/core/knowhere/knowhere/index/preprocessor/preprocessor.h new file mode 100644 index 0000000000000000000000000000000000000000..10f05442703397c1aaf05b431103643b52640037 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/preprocessor/preprocessor.h @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include + +#include "knowhere/common/dataset.h" + + +namespace zilliz { +namespace knowhere { + + +class Preprocessor { + public: + virtual DatasetPtr + Preprocess(const DatasetPtr &input) = 0; +}; + + +using PreprocessorPtr = std::shared_ptr; + + +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/src/knowhere/index/vector_index/cloner.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/cloner.cpp similarity index 91% rename from cpp/src/core/src/knowhere/index/vector_index/cloner.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/cloner.cpp index f5bcfb896dce20b7f58bd4d192204cb6d39b161f..4519def30083d55d2a35afb11e2f24ff30b06026 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/cloner.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/cloner.cpp @@ -17,10 +17,10 @@ #include "knowhere/common/exception.h" -#include "knowhere/index/vector_index/cloner.h" -#include "knowhere/index/vector_index/ivf.h" -#include "knowhere/index/vector_index/gpu_ivf.h" -#include "knowhere/index/vector_index/idmap.h" +#include "cloner.h" +#include "ivf.h" +#include "gpu_ivf.h" +#include "idmap.h" namespace zilliz { diff --git a/cpp/src/core/include/knowhere/index/vector_index/cloner.h b/cpp/src/core/knowhere/knowhere/index/vector_index/cloner.h similarity index 100% rename from cpp/src/core/include/knowhere/index/vector_index/cloner.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/cloner.h diff --git a/cpp/src/core/src/knowhere/index/vector_index/cpu_kdt_rng.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/cpu_kdt_rng.cpp similarity index 83% rename from cpp/src/core/src/knowhere/index/vector_index/cpu_kdt_rng.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/cpu_kdt_rng.cpp index 0805398144e210b42374b1bcbb897e8c38368a5b..d588efb7c6d1100cedc1a0ee09af260dba31a809 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/cpu_kdt_rng.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/cpu_kdt_rng.cpp @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include @@ -7,10 +24,10 @@ #undef mkdir -#include "knowhere/index/vector_index/cpu_kdt_rng.h" -#include "knowhere/index/vector_index/definitions.h" +#include "cpu_kdt_rng.h" +#include "definitions.h" //#include "knowhere/index/preprocessor/normalize.h" -#include "knowhere/index/vector_index/kdt_parameters.h" +#include "kdt_parameters.h" #include "knowhere/adapter/sptag.h" #include "knowhere/common/exception.h" diff --git a/cpp/src/core/include/knowhere/index/vector_index/cpu_kdt_rng.h b/cpp/src/core/knowhere/knowhere/index/vector_index/cpu_kdt_rng.h similarity index 66% rename from cpp/src/core/include/knowhere/index/vector_index/cpu_kdt_rng.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/cpu_kdt_rng.h index 0e1cc55e5df4f4712d9a5633adcb7137a92f7d15..35d488999ea178907e1c4c1ac64fc02da34292ff 100644 --- a/cpp/src/core/include/knowhere/index/vector_index/cpu_kdt_rng.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/cpu_kdt_rng.h @@ -1,8 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #pragma once #include #include -#include "knowhere/index/vector_index/vector_index.h" +#include "vector_index.h" #include "knowhere/index/index_model.h" #include diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/definitions.h b/cpp/src/core/knowhere/knowhere/index/vector_index/definitions.h new file mode 100644 index 0000000000000000000000000000000000000000..3db32e00dc16c2ccd73b0c04a25a4b3499411ab5 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/definitions.h @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include + + +namespace zilliz { +namespace knowhere { + +#define META_ROWS ("rows") +#define META_DIM ("dimension") +#define META_K ("k") + +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/src/knowhere/index/vector_index/gpu_ivf.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp similarity index 99% rename from cpp/src/core/src/knowhere/index/vector_index/gpu_ivf.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp index 590ded16bfc32fbf440d051d09f603cf4fec80f6..fd04501c8304c913d3e6e473e75b1d3b7399e5d6 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/gpu_ivf.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp @@ -27,9 +27,9 @@ #include "knowhere/common/exception.h" -#include "knowhere/index/vector_index/cloner.h" +#include "cloner.h" #include "knowhere/adapter/faiss_adopt.h" -#include "knowhere/index/vector_index/gpu_ivf.h" +#include "gpu_ivf.h" #include diff --git a/cpp/src/core/include/knowhere/index/vector_index/gpu_ivf.h b/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.h similarity index 87% rename from cpp/src/core/include/knowhere/index/vector_index/gpu_ivf.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.h index 8eed6f8fd154aa946e95997bf67f5f62c2fdb9c4..429a94167a8f3a2e10519f58c23e556f81b7cd7a 100644 --- a/cpp/src/core/include/knowhere/index/vector_index/gpu_ivf.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.h @@ -1,3 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #pragma once #include diff --git a/cpp/src/core/src/knowhere/index/vector_index/idmap.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/idmap.cpp similarity index 99% rename from cpp/src/core/src/knowhere/index/vector_index/idmap.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/idmap.cpp index 142d06b5d886bbe660f32a7eb58225c99b9c83ab..b4085e6df8f29d88f283be73b0f43f067b0789c6 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/idmap.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/idmap.cpp @@ -25,7 +25,7 @@ #include "knowhere/common/exception.h" #include "knowhere/adapter/faiss_adopt.h" -#include "knowhere/index/vector_index/idmap.h" +#include "idmap.h" namespace zilliz { diff --git a/cpp/src/core/include/knowhere/index/vector_index/idmap.h b/cpp/src/core/knowhere/knowhere/index/vector_index/idmap.h similarity index 100% rename from cpp/src/core/include/knowhere/index/vector_index/idmap.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/idmap.h diff --git a/cpp/src/core/src/knowhere/index/vector_index/ivf.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/ivf.cpp similarity index 98% rename from cpp/src/core/src/knowhere/index/vector_index/ivf.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/ivf.cpp index acfbb17651998413be36cc951523ee48ebc74ad3..6d72479002bb7702248b5939afa8d240a0873e9f 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/ivf.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/ivf.cpp @@ -25,13 +25,12 @@ #include #include #include -#include #include "knowhere/common/exception.h" -#include "knowhere/index/vector_index/ivf.h" +#include "ivf.h" #include "knowhere/adapter/faiss_adopt.h" -#include "knowhere/index/vector_index/gpu_ivf.h" +#include "gpu_ivf.h" namespace zilliz { diff --git a/cpp/src/core/include/knowhere/index/vector_index/ivf.h b/cpp/src/core/knowhere/knowhere/index/vector_index/ivf.h similarity index 98% rename from cpp/src/core/include/knowhere/index/vector_index/ivf.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/ivf.h index e8ac6cae99b2a5c322c5b8f4a240743227b74acd..c23b5abcc29f1229252662bcab47fcda1c44c814 100644 --- a/cpp/src/core/include/knowhere/index/vector_index/ivf.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/ivf.h @@ -25,7 +25,7 @@ #include #include -#include "knowhere/index/vector_index/vector_index.h" +#include "vector_index.h" namespace zilliz { diff --git a/cpp/src/core/src/knowhere/index/vector_index/kdt_parameters.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.cpp similarity index 54% rename from cpp/src/core/src/knowhere/index/vector_index/kdt_parameters.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.cpp index 9f62b5d3a2bedc9f2c62675d226581c28e0f20a7..b0ed9f6d564b498e1abf3951b6ed6b0b552a418b 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/kdt_parameters.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.cpp @@ -1,6 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include -#include "knowhere/index/vector_index/kdt_parameters.h" + +#include "kdt_parameters.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.h b/cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.h new file mode 100644 index 0000000000000000000000000000000000000000..03f660a0ef8b96dd2bf01c8ba90666a1b0393b70 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.h @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include +#include + + +namespace zilliz { +namespace knowhere { + +using KDTParameter = std::pair; + +class KDTParameterManagement { + public: + const std::vector & + GetKDTParameters(); + + public: + static KDTParameterManagement & + GetInstance() { + static KDTParameterManagement instance; + return instance; + } + + KDTParameterManagement(const KDTParameterManagement &) = delete; + KDTParameterManagement &operator=(const KDTParameterManagement &) = delete; + private: + KDTParameterManagement(); + + private: + std::vector kdt_parameters_; +}; + +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/src/knowhere/index/vector_index/nsg/index_io.h b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/index_io.h similarity index 95% rename from cpp/src/core/src/knowhere/index/vector_index/nsg/index_io.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg/index_io.h index fd8fc9e7ad1f76bfecb6c56d854f17dbb2f9cc7a..60e27bd7f66defb91c938e8be21f937d45396b70 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/nsg/index_io.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/index_io.h @@ -18,7 +18,7 @@ #pragma once -#include "knowhere/index/vector_index/nsg/nsg.h" +#include "nsg.h" namespace zilliz { namespace knowhere { diff --git a/cpp/src/core/include/knowhere/index/vector_index/nsg/neighbor.h b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/neighbor.h similarity index 100% rename from cpp/src/core/include/knowhere/index/vector_index/nsg/neighbor.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg/neighbor.h diff --git a/cpp/src/core/src/knowhere/index/vector_index/nsg/nsg.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg.cpp similarity index 99% rename from cpp/src/core/src/knowhere/index/vector_index/nsg/nsg.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg.cpp index d3efc5e38ae8f55af57a50a76d5e5f2e9f96f103..d42f4c566241081aa4a9f02ef6e515b783db20f9 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/nsg/nsg.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg.cpp @@ -22,7 +22,7 @@ #include #include -#include "knowhere/index/vector_index/nsg/nsg.h" +#include "nsg.h" #include "knowhere/common/exception.h" #include "knowhere/common/timer.h" #include "utils.h" diff --git a/cpp/src/core/include/knowhere/index/vector_index/nsg/nsg.h b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg.h similarity index 100% rename from cpp/src/core/include/knowhere/index/vector_index/nsg/nsg.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg.h diff --git a/cpp/src/core/src/knowhere/index/vector_index/nsg/nsg_io.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg_io.cpp similarity index 97% rename from cpp/src/core/src/knowhere/index/vector_index/nsg/nsg_io.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg_io.cpp index 283e8cc7e2f75a699665b62c0f38905169c32346..34d629ec4b10115690967cb6e455726fe0d2cba7 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/nsg/nsg_io.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg_io.cpp @@ -18,7 +18,7 @@ #include -#include "knowhere/index/vector_index/nsg/nsg_io.h" +#include "nsg_io.h" namespace zilliz { diff --git a/cpp/src/core/include/knowhere/index/vector_index/nsg/nsg_io.h b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg_io.h similarity index 100% rename from cpp/src/core/include/knowhere/index/vector_index/nsg/nsg_io.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg_io.h diff --git a/cpp/src/core/src/knowhere/index/vector_index/nsg/utils.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/utils.cpp similarity index 100% rename from cpp/src/core/src/knowhere/index/vector_index/nsg/utils.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg/utils.cpp diff --git a/cpp/src/core/src/knowhere/index/vector_index/nsg/utils.h b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/utils.h similarity index 95% rename from cpp/src/core/src/knowhere/index/vector_index/nsg/utils.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg/utils.h index 82c8a00dc8ee4b04e128ecb5bf514ea14cc021a9..a57a7738fda3a346184fc9df3bd3c807d9994eb5 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/nsg/utils.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/utils.h @@ -23,7 +23,7 @@ #include -#include "knowhere/index/vector_index/nsg/nsg.h" +#include "nsg.h" #include "knowhere/common/config.h" diff --git a/cpp/src/core/src/knowhere/index/vector_index/nsg_index.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg_index.cpp similarity index 96% rename from cpp/src/core/src/knowhere/index/vector_index/nsg_index.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg_index.cpp index 550efef70d792a88cdca86a32021cd4cb2895293..724c80dfcaf89a6c69f82d5317973761a4cd0d46 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/nsg_index.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg_index.cpp @@ -16,12 +16,12 @@ // under the License. -#include "knowhere/index/vector_index/nsg_index.h" +#include "nsg_index.h" #include "knowhere/index/vector_index/nsg/nsg.h" #include "knowhere/index/vector_index/nsg/nsg_io.h" -#include "knowhere/index/vector_index/idmap.h" -#include "knowhere/index/vector_index/ivf.h" -#include "knowhere/index/vector_index/gpu_ivf.h" +#include "idmap.h" +#include "ivf.h" +#include "gpu_ivf.h" #include "knowhere/adapter/faiss_adopt.h" #include "knowhere/common/exception.h" #include "knowhere/common/timer.h" diff --git a/cpp/src/core/include/knowhere/index/vector_index/nsg_index.h b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg_index.h similarity index 96% rename from cpp/src/core/include/knowhere/index/vector_index/nsg_index.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg_index.h index a0829f381cc2a3585d0cc7dca9ce95a4f102a7d0..e63dbb03b4fec5b906b391449751c3875383e6c1 100644 --- a/cpp/src/core/include/knowhere/index/vector_index/nsg_index.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg_index.h @@ -18,7 +18,7 @@ #pragma once -#include "knowhere/index/vector_index/vector_index.h" +#include "vector_index.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/vector_index.h b/cpp/src/core/knowhere/knowhere/index/vector_index/vector_index.h new file mode 100644 index 0000000000000000000000000000000000000000..1828e3573d30c595a24ebc55981926e4dbe02c97 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/vector_index.h @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + + +#include + +#include "knowhere/common/config.h" +#include "knowhere/common/dataset.h" +#include "knowhere/index/index.h" +#include "knowhere/index/preprocessor/preprocessor.h" + + +namespace zilliz { +namespace knowhere { + +class VectorIndex; +using VectorIndexPtr = std::shared_ptr; + + +class VectorIndex : public Index { + public: + virtual PreprocessorPtr + BuildPreprocessor(const DatasetPtr &dataset, const Config &config) { return nullptr; } + + virtual IndexModelPtr + Train(const DatasetPtr &dataset, const Config &config) { return nullptr; } + + virtual void + Add(const DatasetPtr &dataset, const Config &config) = 0; + + virtual void + Seal() = 0; + + virtual VectorIndexPtr + Clone() = 0; + + virtual int64_t + Count() = 0; + + virtual int64_t + Dimension() = 0; +}; + + + +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/test/CMakeLists.txt b/cpp/src/core/test/CMakeLists.txt index cba8a03b94771e54f229cb30c117d8703bd8ec1a..5b8dca2e4b01cf32b5eee2759e92205fd95fb357 100644 --- a/cpp/src/core/test/CMakeLists.txt +++ b/cpp/src/core/test/CMakeLists.txt @@ -1,6 +1,6 @@ include_directories(${CORE_SOURCE_DIR}/thirdparty) include_directories(${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService) -include_directories(${CORE_SOURCE_DIR}/include) +include_directories(${CORE_SOURCE_DIR}/knowhere) include_directories(${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include) include_directories(/usr/local/cuda/include) link_directories(/usr/local/cuda/lib64) @@ -25,13 +25,13 @@ set(basic_libs # set(ivf_srcs - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp - ${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp - ${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp - ${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/ivf.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu_ivf.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/cloner.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/idmap.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/structure.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/common/exception.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/common/timer.cpp utils.cpp ) if(NOT TARGET test_ivf) @@ -41,13 +41,13 @@ target_link_libraries(test_ivf ${depend_libs} ${unittest_libs} ${basic_libs}) # set(idmap_srcs - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp - ${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp - ${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp - ${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/idmap.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/ivf.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/cloner.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu_ivf.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/structure.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/common/exception.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/common/timer.cpp utils.cpp ) if(NOT TARGET test_idmap) @@ -57,14 +57,14 @@ target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs}) # set(kdt_srcs - ${CORE_SOURCE_DIR}/src/knowhere/index/preprocessor/normalize.cpp - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/kdt_parameters.cpp - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cpu_kdt_rng.cpp - ${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp - ${CORE_SOURCE_DIR}/src/knowhere/adapter/sptag.cpp - ${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp - ${CORE_SOURCE_DIR}/src/knowhere/adapter/arrow.cpp - ${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/preprocessor/normalize.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/kdt_parameters.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/cpu_kdt_rng.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/structure.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/sptag.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/common/exception.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/arrow.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/common/timer.cpp utils.cpp ) if(NOT TARGET test_kdt)