提交 ed105628 编写于 作者: X xiaojun.lin

MS-565 1. refactor dir structure 2. add license 3. del swig


Former-commit-id: 580d35b9fe3ea71ebc7d6096fbf5d905351f34f5
上级 885f9ec1
......@@ -84,7 +84,7 @@ include(DefineOptionsCore)
include(BuildUtilsCore)
include(ThirdPartyPackagesCore)
add_subdirectory(src)
add_subdirectory(knowhere)
if (BUILD_COVERAGE STREQUAL "ON")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
......
#pragma once
#include <memory>
#include <knowhere/common/array.h>
namespace zilliz {
namespace knowhere {
ArrayPtr
CopyArray(const ArrayPtr &origin);
SchemaPtr
CopySchema(const SchemaPtr &origin);
} // namespace knowhere
} // namespace zilliz
#pragma once
#include <memory>
#include <knowhere/common/dataset.h>
#include <SPTAG/AnnService/inc/Core/VectorIndex.h>
namespace zilliz {
namespace knowhere {
std::shared_ptr<SPTAG::VectorSet>
ConvertToVectorSet(const DatasetPtr &dataset);
std::shared_ptr<SPTAG::MetadataSet>
ConvertToMetadataSet(const DatasetPtr &dataset);
std::vector<SPTAG::QueryResult>
ConvertToQueryResult(const DatasetPtr &dataset, const Config &config);
DatasetPtr
ConvertToDataset(std::vector<SPTAG::QueryResult> query_results);
} // namespace knowhere
} // namespace zilliz
#pragma once
#include <jsoncons/json.hpp>
namespace zilliz {
namespace knowhere {
using Config = jsoncons::json;
} // namespace knowhere
} // namespace zilliz
#pragma once
namespace zilliz {
namespace sched {
namespace master {
} // namespace master
} // namespace sched
} // namespace zilliz
#pragma once
#include <cstdint>
#include "zlibrary/error/error.h"
namespace zilliz {
namespace knowhere {
using Error = zilliz::lib::ErrorCode;
constexpr Error STORE_SUCCESS = zilliz::lib::SUCCESS_CODE;
constexpr Error ERROR_CODE_BASE = 0x36000;
constexpr Error ERROR_CODE_END = 0x37000;
constexpr Error
ToGlobalErrorCode(const Error error_code) {
return zilliz::lib::ToGlobalErrorCode(error_code, ERROR_CODE_BASE);
}
class Exception : public zilliz::lib::Exception {
public:
Exception(const Error error_code,
const std::string &message = nullptr)
: zilliz::lib::Exception(error_code, "KNOWHERE", message) {}
};
constexpr Error UNEXPECTED = ToGlobalErrorCode(0x001);
constexpr Error UNSUPPORTED = ToGlobalErrorCode(0x002);
constexpr Error NULL_POINTER = ToGlobalErrorCode(0x003);
constexpr Error OVERFLOW = ToGlobalErrorCode(0x004);
constexpr Error INVALID_ARGUMENT = ToGlobalErrorCode(0x005);
constexpr Error UNSUPPORTED_TYPE = ToGlobalErrorCode(0x006);
} // namespace store
} // namespace zilliz
using Error = zilliz::store::Error;
#pragma once
//#include "zcommon/id/id.h"
//using ID = zilliz::common::ID;
#include <stdint.h>
#include <string>
namespace zilliz {
namespace knowhere {
class ID {
public:
constexpr static int64_t kIDSize = 20;
public:
const int32_t *
data() const { return content_; }
int32_t *
mutable_data() { return content_; }
bool
IsValid() const;
std::string
ToString() const;
bool
operator==(const ID &that) const;
bool
operator<(const ID &that) const;
protected:
int32_t content_[5] = {};
};
} // namespace knowhere
} // namespace zilliz
#pragma once
#include <memory>
#include "arrow/type.h"
namespace zilliz {
namespace knowhere {
using DataType = arrow::DataType;
using Field = arrow::Field;
using FieldPtr = std::shared_ptr<arrow::Field>;
using Schema = arrow::Schema;
using SchemaPtr = std::shared_ptr<Schema>;
using SchemaConstPtr = std::shared_ptr<const Schema>;
} // namespace knowhere
} // namespace zilliz
#pragma once
#include <memory>
#include "arrow/tensor.h"
namespace zilliz {
namespace knowhere {
using Tensor = arrow::Tensor;
using TensorPtr = std::shared_ptr<Tensor>;
} // namespace knowhere
} // namespace zilliz
#pragma once
#include <memory>
#include "knowhere/common/binary_set.h"
#include "knowhere/common/dataset.h"
#include "knowhere/index/index_type.h"
#include "knowhere/index/index_model.h"
#include "knowhere/index/preprocessor/preprocessor.h"
namespace zilliz {
namespace knowhere {
class Index {
public:
virtual BinarySet
Serialize() = 0;
virtual void
Load(const BinarySet &index_binary) = 0;
// @throw
virtual DatasetPtr
Search(const DatasetPtr &dataset, const Config &config) = 0;
public:
IndexType
idx_type() const { return idx_type_; }
void
set_idx_type(IndexType idx_type) { idx_type_ = idx_type; }
virtual void
set_preprocessor(PreprocessorPtr preprocessor) {}
virtual void
set_index_model(IndexModelPtr model) {}
private:
IndexType idx_type_;
};
using IndexPtr = std::shared_ptr<Index>;
} // namespace knowhere
} // namespace zilliz
#pragma once
#include <memory>
#include "knowhere/common/binary_set.h"
namespace zilliz {
namespace knowhere {
class IndexModel {
public:
virtual BinarySet
Serialize() = 0;
virtual void
Load(const BinarySet &binary) = 0;
};
using IndexModelPtr = std::shared_ptr<IndexModel>;
} // namespace knowhere
} // namespace zilliz
#pragma once
namespace zilliz {
namespace knowhere {
enum class IndexType {
kUnknown = 0,
kVecIdxBegin = 100,
kVecIVFFlat = kVecIdxBegin,
kVecIdxEnd,
};
} // namespace knowhere
} // namespace zilliz
#pragma once
#include <memory>
#include "knowhere/common/dataset.h"
namespace zilliz {
namespace knowhere {
class Preprocessor {
public:
virtual DatasetPtr
Preprocess(const DatasetPtr &input) = 0;
};
using PreprocessorPtr = std::shared_ptr<Preprocessor>;
} // namespace knowhere
} // namespace zilliz
#pragma once
#include <string>
namespace zilliz {
namespace knowhere {
#define META_ROWS ("rows")
#define META_DIM ("dimension")
#define META_K ("k")
} // namespace knowhere
} // namespace zilliz
#pragma once
#include <string>
#include <vector>
namespace zilliz {
namespace knowhere {
using KDTParameter = std::pair<std::string, std::string>;
class KDTParameterManagement {
public:
const std::vector<KDTParameter> &
GetKDTParameters();
public:
static KDTParameterManagement &
GetInstance() {
static KDTParameterManagement instance;
return instance;
}
KDTParameterManagement(const KDTParameterManagement &) = delete;
KDTParameterManagement &operator=(const KDTParameterManagement &) = delete;
private:
KDTParameterManagement();
private:
std::vector<KDTParameter> kdt_parameters_;
};
} // namespace knowhere
} // namespace zilliz
%module nsg
%{
#define SWIG_FILE_WITH_INIT
#include <numpy/arrayobject.h>
/* Include the header in the wrapper code */
#include "nsg.h"
%}
/* Parse the header file */
%include "index.h"
#pragma once
#include <memory>
#include "knowhere/common/config.h"
#include "knowhere/common/dataset.h"
#include "knowhere/index/index.h"
#include "knowhere/index/preprocessor/preprocessor.h"
namespace zilliz {
namespace knowhere {
class VectorIndex;
using VectorIndexPtr = std::shared_ptr<VectorIndex>;
class VectorIndex : public Index {
public:
virtual PreprocessorPtr
BuildPreprocessor(const DatasetPtr &dataset, const Config &config) { return nullptr; }
virtual IndexModelPtr
Train(const DatasetPtr &dataset, const Config &config) { return nullptr; }
virtual void
Add(const DatasetPtr &dataset, const Config &config) = 0;
virtual void
Seal() = 0;
virtual VectorIndexPtr
Clone() = 0;
virtual int64_t
Count() = 0;
virtual int64_t
Dimension() = 0;
};
} // namespace knowhere
} // namespace zilliz
......@@ -5,7 +5,7 @@ include_directories(${TBB_DIR}/include)
include_directories(${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
link_directories(${CUDA_TOOLKIT_ROOT_DIR}/lib64)
include_directories(${CORE_SOURCE_DIR}/include)
include_directories(${CORE_SOURCE_DIR}/knowhere)
include_directories(${CORE_SOURCE_DIR}/thirdparty)
include_directories(${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService)
include_directories(${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include)
......@@ -107,7 +107,7 @@ INSTALL(FILES ${CORE_SOURCE_DIR}/thirdparty/tbb/libtbb.so
)
set(CORE_INCLUDE_DIRS
${CORE_SOURCE_DIR}/include
${CORE_SOURCE_DIR}/knowhere
${CORE_SOURCE_DIR}/thirdparty
${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService
${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include
......
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "knowhere/adapter/arrow.h"
#include "arrow.h"
namespace zilliz {
namespace knowhere {
......
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <memory>
#include "knowhere/common/array.h"
namespace zilliz {
namespace knowhere {
ArrayPtr
CopyArray(const ArrayPtr &origin);
SchemaPtr
CopySchema(const SchemaPtr &origin);
} // namespace knowhere
} // namespace zilliz
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "knowhere/index/vector_index/definitions.h"
#include "knowhere/adapter/sptag.h"
#include "knowhere/adapter/structure.h"
#include "sptag.h"
#include "structure.h"
namespace zilliz {
......
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <memory>
#include <SPTAG/AnnService/inc/Core/VectorIndex.h>
#include "knowhere/common/dataset.h"
namespace zilliz {
namespace knowhere {
std::shared_ptr<SPTAG::VectorSet>
ConvertToVectorSet(const DatasetPtr &dataset);
std::shared_ptr<SPTAG::MetadataSet>
ConvertToMetadataSet(const DatasetPtr &dataset);
std::vector<SPTAG::QueryResult>
ConvertToQueryResult(const DatasetPtr &dataset, const Config &config);
DatasetPtr
ConvertToDataset(std::vector<SPTAG::QueryResult> query_results);
} // namespace knowhere
} // namespace zilliz
......@@ -16,7 +16,7 @@
// under the License.
#include "knowhere/adapter/structure.h"
#include "structure.h"
namespace zilliz {
......
......@@ -19,7 +19,7 @@
#pragma once
#include <memory>
#include <knowhere/common/dataset.h>
#include "knowhere/common/dataset.h"
namespace zilliz {
......
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "arrow/array.h"
#include "knowhere/common/schema.h"
#include "schema.h"
namespace zilliz {
......
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <map>
......@@ -5,7 +23,7 @@
#include <vector>
#include <memory>
#include "knowhere/common/id.h"
#include "id.h"
namespace zilliz {
......
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <memory>
#include "arrow/buffer.h"
......
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <jsoncons/json.hpp>
namespace zilliz {
namespace knowhere {
using Config = jsoncons::json;
} // namespace knowhere
} // namespace zilliz
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <vector>
#include <memory>
#include "knowhere/common/array.h"
#include "knowhere/common/buffer.h"
#include "knowhere/common/tensor.h"
#include "knowhere/common/schema.h"
#include "knowhere/common/config.h"
#include "array.h"
#include "buffer.h"
#include "tensor.h"
#include "schema.h"
#include "config.h"
#include "knowhere/adapter/arrow.h"
......
......@@ -16,9 +16,10 @@
// under the License.
#include "knowhere/common/exception.h"
#include <cstdio>
#include "exception.h"
namespace zilliz {
namespace knowhere {
......
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
//#include "zcommon/id/id.h"
//using ID = zilliz::common::ID;
#include <stdint.h>
#include <string>
namespace zilliz {
namespace knowhere {
class ID {
public:
constexpr static int64_t kIDSize = 20;
public:
const int32_t *
data() const { return content_; }
int32_t *
mutable_data() { return content_; }
bool
IsValid() const;
std::string
ToString() const;
bool
operator==(const ID &that) const;
bool
operator<(const ID &that) const;
protected:
int32_t content_[5] = {};
};
} // namespace knowhere
} // namespace zilliz
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <memory>
#include "arrow/type.h"
namespace zilliz {
namespace knowhere {
using DataType = arrow::DataType;
using Field = arrow::Field;
using FieldPtr = std::shared_ptr<arrow::Field>;
using Schema = arrow::Schema;
using SchemaPtr = std::shared_ptr<Schema>;
using SchemaConstPtr = std::shared_ptr<const Schema>;
} // namespace knowhere
} // namespace zilliz
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <memory>
#include "arrow/tensor.h"
namespace zilliz {
namespace knowhere {
using Tensor = arrow::Tensor;
using TensorPtr = std::shared_ptr<Tensor>;
} // namespace knowhere
} // namespace zilliz
......@@ -18,7 +18,7 @@
#include <iostream> // TODO(linxj): using Log instead
#include "knowhere/common/timer.h"
#include "timer.h"
namespace zilliz {
namespace knowhere {
......
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <memory>
#include "knowhere/common/binary_set.h"
#include "knowhere/common/dataset.h"
#include "index_type.h"
#include "index_model.h"
#include "knowhere/index/preprocessor/preprocessor.h"
namespace zilliz {
namespace knowhere {
class Index {
public:
virtual BinarySet
Serialize() = 0;
virtual void
Load(const BinarySet &index_binary) = 0;
// @throw
virtual DatasetPtr
Search(const DatasetPtr &dataset, const Config &config) = 0;
public:
IndexType
idx_type() const { return idx_type_; }
void
set_idx_type(IndexType idx_type) { idx_type_ = idx_type; }
virtual void
set_preprocessor(PreprocessorPtr preprocessor) {}
virtual void
set_index_model(IndexModelPtr model) {}
private:
IndexType idx_type_;
};
using IndexPtr = std::shared_ptr<Index>;
} // namespace knowhere
} // namespace zilliz
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <memory>
#include "knowhere/common/binary_set.h"
namespace zilliz {
namespace knowhere {
class IndexModel {
public:
virtual BinarySet
Serialize() = 0;
virtual void
Load(const BinarySet &binary) = 0;
};
using IndexModelPtr = std::shared_ptr<IndexModel>;
} // namespace knowhere
} // namespace zilliz
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
namespace zilliz {
namespace knowhere {
enum class IndexType {
kUnknown = 0,
kVecIdxBegin = 100,
kVecIVFFlat = kVecIdxBegin,
kVecIdxEnd,
};
} // namespace knowhere
} // namespace zilliz
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <memory>
#include "knowhere/common/dataset.h"
namespace zilliz {
namespace knowhere {
class Preprocessor {
public:
virtual DatasetPtr
Preprocess(const DatasetPtr &input) = 0;
};
using PreprocessorPtr = std::shared_ptr<Preprocessor>;
} // namespace knowhere
} // namespace zilliz
......@@ -17,10 +17,10 @@
#include "knowhere/common/exception.h"
#include "knowhere/index/vector_index/cloner.h"
#include "knowhere/index/vector_index/ivf.h"
#include "knowhere/index/vector_index/gpu_ivf.h"
#include "knowhere/index/vector_index/idmap.h"
#include "cloner.h"
#include "ivf.h"
#include "gpu_ivf.h"
#include "idmap.h"
namespace zilliz {
......
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <sstream>
#include <SPTAG/AnnService/inc/Server/QueryParser.h>
......@@ -7,10 +24,10 @@
#undef mkdir
#include "knowhere/index/vector_index/cpu_kdt_rng.h"
#include "knowhere/index/vector_index/definitions.h"
#include "cpu_kdt_rng.h"
#include "definitions.h"
//#include "knowhere/index/preprocessor/normalize.h"
#include "knowhere/index/vector_index/kdt_parameters.h"
#include "kdt_parameters.h"
#include "knowhere/adapter/sptag.h"
#include "knowhere/common/exception.h"
......
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <cstdint>
#include <memory>
#include "knowhere/index/vector_index/vector_index.h"
#include "vector_index.h"
#include "knowhere/index/index_model.h"
#include <SPTAG/AnnService/inc/Core/VectorIndex.h>
......
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <string>
namespace zilliz {
namespace knowhere {
#define META_ROWS ("rows")
#define META_DIM ("dimension")
#define META_K ("k")
} // namespace knowhere
} // namespace zilliz
......@@ -27,9 +27,9 @@
#include "knowhere/common/exception.h"
#include "knowhere/index/vector_index/cloner.h"
#include "cloner.h"
#include "knowhere/adapter/faiss_adopt.h"
#include "knowhere/index/vector_index/gpu_ivf.h"
#include "gpu_ivf.h"
#include <algorithm>
......
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <faiss/gpu/StandardGpuResources.h>
......
......@@ -25,7 +25,7 @@
#include "knowhere/common/exception.h"
#include "knowhere/adapter/faiss_adopt.h"
#include "knowhere/index/vector_index/idmap.h"
#include "idmap.h"
namespace zilliz {
......
......@@ -25,13 +25,12 @@
#include <faiss/index_io.h>
#include <faiss/gpu/StandardGpuResources.h>
#include <faiss/gpu/GpuAutoTune.h>
#include <knowhere/index/vector_index/ivf.h>
#include "knowhere/common/exception.h"
#include "knowhere/index/vector_index/ivf.h"
#include "ivf.h"
#include "knowhere/adapter/faiss_adopt.h"
#include "knowhere/index/vector_index/gpu_ivf.h"
#include "gpu_ivf.h"
namespace zilliz {
......
......@@ -25,7 +25,7 @@
#include <faiss/AuxIndexStructures.h>
#include <faiss/Index.h>
#include "knowhere/index/vector_index/vector_index.h"
#include "vector_index.h"
namespace zilliz {
......
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <mutex>
#include "knowhere/index/vector_index/kdt_parameters.h"
#include "kdt_parameters.h"
namespace zilliz {
......
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <string>
#include <vector>
namespace zilliz {
namespace knowhere {
using KDTParameter = std::pair<std::string, std::string>;
class KDTParameterManagement {
public:
const std::vector<KDTParameter> &
GetKDTParameters();
public:
static KDTParameterManagement &
GetInstance() {
static KDTParameterManagement instance;
return instance;
}
KDTParameterManagement(const KDTParameterManagement &) = delete;
KDTParameterManagement &operator=(const KDTParameterManagement &) = delete;
private:
KDTParameterManagement();
private:
std::vector<KDTParameter> kdt_parameters_;
};
} // namespace knowhere
} // namespace zilliz
......@@ -18,7 +18,7 @@
#pragma once
#include "knowhere/index/vector_index/nsg/nsg.h"
#include "nsg.h"
namespace zilliz {
namespace knowhere {
......
......@@ -22,7 +22,7 @@
#include <stack>
#include <omp.h>
#include "knowhere/index/vector_index/nsg/nsg.h"
#include "nsg.h"
#include "knowhere/common/exception.h"
#include "knowhere/common/timer.h"
#include "utils.h"
......
......@@ -18,7 +18,7 @@
#include <cstring>
#include "knowhere/index/vector_index/nsg/nsg_io.h"
#include "nsg_io.h"
namespace zilliz {
......
......@@ -23,7 +23,7 @@
#include <faiss/AutoTune.h>
#include "knowhere/index/vector_index/nsg/nsg.h"
#include "nsg.h"
#include "knowhere/common/config.h"
......
......@@ -16,12 +16,12 @@
// under the License.
#include "knowhere/index/vector_index/nsg_index.h"
#include "nsg_index.h"
#include "knowhere/index/vector_index/nsg/nsg.h"
#include "knowhere/index/vector_index/nsg/nsg_io.h"
#include "knowhere/index/vector_index/idmap.h"
#include "knowhere/index/vector_index/ivf.h"
#include "knowhere/index/vector_index/gpu_ivf.h"
#include "idmap.h"
#include "ivf.h"
#include "gpu_ivf.h"
#include "knowhere/adapter/faiss_adopt.h"
#include "knowhere/common/exception.h"
#include "knowhere/common/timer.h"
......
......@@ -18,7 +18,7 @@
#pragma once
#include "knowhere/index/vector_index/vector_index.h"
#include "vector_index.h"
namespace zilliz {
......
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <memory>
#include "knowhere/common/config.h"
#include "knowhere/common/dataset.h"
#include "knowhere/index/index.h"
#include "knowhere/index/preprocessor/preprocessor.h"
namespace zilliz {
namespace knowhere {
class VectorIndex;
using VectorIndexPtr = std::shared_ptr<VectorIndex>;
class VectorIndex : public Index {
public:
virtual PreprocessorPtr
BuildPreprocessor(const DatasetPtr &dataset, const Config &config) { return nullptr; }
virtual IndexModelPtr
Train(const DatasetPtr &dataset, const Config &config) { return nullptr; }
virtual void
Add(const DatasetPtr &dataset, const Config &config) = 0;
virtual void
Seal() = 0;
virtual VectorIndexPtr
Clone() = 0;
virtual int64_t
Count() = 0;
virtual int64_t
Dimension() = 0;
};
} // namespace knowhere
} // namespace zilliz
include_directories(${CORE_SOURCE_DIR}/thirdparty)
include_directories(${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService)
include_directories(${CORE_SOURCE_DIR}/include)
include_directories(${CORE_SOURCE_DIR}/knowhere)
include_directories(${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include)
include_directories(/usr/local/cuda/include)
link_directories(/usr/local/cuda/lib64)
......@@ -25,13 +25,13 @@ set(basic_libs
#<IVF-TEST>
set(ivf_srcs
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp
${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp
${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/ivf.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu_ivf.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/cloner.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/idmap.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/structure.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/common/exception.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/common/timer.cpp
utils.cpp
)
if(NOT TARGET test_ivf)
......@@ -41,13 +41,13 @@ target_link_libraries(test_ivf ${depend_libs} ${unittest_libs} ${basic_libs})
#<IDMAP-TEST>
set(idmap_srcs
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp
${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp
${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/idmap.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/ivf.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/cloner.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu_ivf.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/structure.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/common/exception.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/common/timer.cpp
utils.cpp
)
if(NOT TARGET test_idmap)
......@@ -57,14 +57,14 @@ target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs})
#<KDT-TEST>
set(kdt_srcs
${CORE_SOURCE_DIR}/src/knowhere/index/preprocessor/normalize.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/kdt_parameters.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cpu_kdt_rng.cpp
${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
${CORE_SOURCE_DIR}/src/knowhere/adapter/sptag.cpp
${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp
${CORE_SOURCE_DIR}/src/knowhere/adapter/arrow.cpp
${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/preprocessor/normalize.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/kdt_parameters.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/cpu_kdt_rng.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/structure.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/sptag.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/common/exception.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/arrow.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/common/timer.cpp
utils.cpp
)
if(NOT TARGET test_kdt)
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册