Skip to content

milvus
milvus

qq_35024513 / milvus
与 Fork 源项目一致

Fork自 milvus / milvus

通知 14
Star 0
Fork 0
milvus
milvus

体验新版 GitCode，发现更多精彩内容 >>

提交 3c6f49f9 编写于 作者: W wxyu
浏览文件
操作

MS-555 Remove old scheduler 


Former-commit-id: e07f211e59232b04cfa3f37240e896461ad9b616
上级 e72b0348 aec55278
展开全部 隐藏空白更改
内联 并排
Showing with 2237 addition and 3859 deletion
cpp/CHANGELOG.md
浏览文件 @ 3c6f49f9
......@@ -13,12 +13,15 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-556 - Add Job Definition in Scheduler
- MS-558 - Refine status code
- MS-562 - Add JobMgr and TaskCreator in Scheduler
- MS-566 - Refactor cmake
- MS-555 - Remove old scheduler
## New Feature
## Task
- MS-554 - Change license to Apache 2.0
- MS-561 - Add contributing guidelines, code of conduct and README docs
- MS-567 - Add NOTICE.md
# Milvus 0.4.0 (2019-09-12)
......
cpp/CMakeLists.txt
浏览文件 @ 3c6f49f9
......@@ -42,24 +42,15 @@ set(MILVUS_VERSION "${GIT_BRANCH_NAME}")
string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]" MILVUS_VERSION "${MILVUS_VERSION}")
if(CMAKE_BUILD_TYPE STREQUAL "Release")
set(BUILD_TYPE "release")
set(BUILD_TYPE "Release")
else()
set(BUILD_TYPE "debug")
set(BUILD_TYPE "Debug")
endif()
message(STATUS "Build type = ${BUILD_TYPE}")
add_definitions(-DNEW_SCHEDULER)
project(milvus VERSION "${MILVUS_VERSION}")
project(milvus_engine LANGUAGES CUDA CXX)
# Ensure that a default make is set
if("${MAKE}" STREQUAL "")
if(NOT MSVC)
find_program(MAKE make)
endif()
endif()
set(MILVUS_VERSION_MAJOR "${milvus_VERSION_MAJOR}")
set(MILVUS_VERSION_MINOR "${milvus_VERSION_MINOR}")
set(MILVUS_VERSION_PATCH "${milvus_VERSION_PATCH}")
......@@ -78,86 +69,78 @@ message(STATUS "Milvus version: "
"${MILVUS_VERSION_MAJOR}.${MILVUS_VERSION_MINOR}.${MILVUS_VERSION_PATCH} "
"(full: '${MILVUS_VERSION}')")
set(MILVUS_SOURCE_DIR ${PROJECT_SOURCE_DIR})
set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR})
find_package(CUDA)
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES -arch sm_60 --expt-extended-lambda")
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g")
message(STATUS "CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR}")
message(STATUS "CUDA_NVCC_FLAGS=${CUDA_NVCC_FLAGS}")
set(CMAKE_CXX_STANDARD 14)
set(CMAKE_CXX_STANDARD_REQUIRED on)
if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
message(STATUS "building milvus_engine on x86 architecture")
message(STATUS "Building milvus_engine on x86 architecture")
set(MILVUS_BUILD_ARCH x86_64)
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(ppc)")
message(STATUS "building milvus_engine on ppc architecture")
message(STATUS "Building milvus_engine on ppc architecture")
set(MILVUS_BUILD_ARCH ppc64le)
else()
message(WARNING "unknown processor type")
message(WARNING "Unknown processor type")
message(WARNING "CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}")
set(MILVUS_BUILD_ARCH unknown)
endif()
find_package(CUDA)
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES --expt-extended-lambda")
if(CMAKE_BUILD_TYPE STREQUAL "Release")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp")
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3")
else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp")
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g")
endif()
# Ensure that a default make is set
if("${MAKE}" STREQUAL "")
if(NOT MSVC)
find_program(MAKE make)
endif()
endif()
set(ALLOW_DUPLICATE_CUSTOM_TARGETS TRUE)
find_path(MYSQL_INCLUDE_DIR
NAMES "mysql.h"
PATH_SUFFIXES "mysql")
if (${MYSQL_INCLUDE_DIR} STREQUAL "MYSQL_INCLUDE_DIR-NOTFOUND")
message(FATAL_ERROR "Could not found MySQL include directory")
else()
include_directories(${MYSQL_INCLUDE_DIR})
endif()
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
set(MILVUS_SOURCE_DIR ${PROJECT_SOURCE_DIR})
set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR})
set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src)
include(ExternalProject)
include(DefineOptions)
include(BuildUtils)
include(ThirdPartyPackages)
include_directories(${MILVUS_SOURCE_DIR})
link_directories(${MILVUS_BINARY_DIR})
## Following should be check
set(MILVUS_ENGINE_INCLUDE ${PROJECT_SOURCE_DIR}/include)
set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src)
add_compile_definitions(PROFILER=${PROFILER})
message(STATUS "MILVUS_ENABLE_PROFILING = ${MILVUS_ENABLE_PROFILING}")
if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
ADD_DEFINITIONS(-DMILVUS_ENABLE_PROFILING)
endif()
include_directories(${MILVUS_ENGINE_INCLUDE})
include_directories(${MILVUS_ENGINE_SRC})
link_directories(${CMAKE_CURRRENT_BINARY_DIR})
config_summary()
add_subdirectory(src)
if (BUILD_COVERAGE STREQUAL "ON")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
endif()
if (BUILD_UNIT_TEST STREQUAL "ON")
if (BUILD_COVERAGE STREQUAL "ON")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
endif()
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/unittest)
endif()
add_custom_target(Clean-All COMMAND ${CMAKE_BUILD_TOOL} clean)
if("${MILVUS_DB_PATH}" STREQUAL "")
set(MILVUS_DB_PATH "/tmp/milvus")
endif()
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.conf)
#install
install(DIRECTORY scripts/
DESTINATION scripts
FILE_PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ
......@@ -169,12 +152,3 @@ install(FILES
conf/log_config.conf
DESTINATION
conf)
install(FILES
./Milvus-EULA-cn.md
./Milvus-EULA-en.md
DESTINATION
license
)
config_summary()
cpp/CODE OF CONDUCT.md 0 → 100644
浏览文件 @ 3c6f49f9
# Milvus Code of Conduct
In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
## Our Standards
Examples of behavior that contributes to creating a positive environment include:
- Using welcoming and inclusive language.
- Being respectful of differing viewpoints and experiences.
- Gracefully accepting constructive criticism.
- Focusing on what is best for the community.
- Showing empathy towards other community members.
Examples of unacceptable behavior by participants include:
- The use of sexualized language or imagery and unwelcome sexual attention or advances.
- Trolling, insulting/derogatory comments, and personal or political attacks.
- Public or private harassment.
- Publishing others' private information, such as a physical or electronic address, without explicit permission.
- Conduct which could reasonably be considered inappropriate for the forum in which it occurs.
All Milvus forums and spaces are meant for professional interactions, and any behavior which could reasonably be considered inappropriate in a professional setting is unacceptable.
## Our Responsibilities
Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
## Scope
This Code of Conduct applies to all content on milvus.io, Milvus’s GitHub organization, or any other official Milvus web presence allowing for community interactions, as well as at all official Milvus events, whether offline or online.
The Code of Conduct also applies within all project spaces and in public spaces whenever an individual is representing Milvus or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed or de facto representative at an online or offline event. Representation of Milvus may be further defined and clarified by project maintainers.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at support@zilliz.com. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org/), version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
For answers to common questions about this code of conduct, see https://www.contributor-covenant.org/faq
cpp/CONTRIBUTING.md 0 → 100644
浏览文件 @ 3c6f49f9
# Contributing to Milvus
First of all, thanks for taking the time to contribute to Milvus! It's people like you that help Milvus come to fruition.
The following are a set of guidelines for contributing to Milvus. Following these guidelines helps contributing to this project easy and transparent. These are mostly guideline, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request.
As for everything else in the project, the contributions to Milvus are governed by our [Code of Conduct](http://hood.ie/code-of-conduct/).
TOC
## Contribution Checklist
Before you make any contributions, make sure you follow this list.
- Read [Contributing to Milvus](CONTRIBUTING.md).
- Check if the changes are consistent with the [coding style](CONTRIBUTING.md#coding-style).
- Run [unit tests](CONTRIBUTING.md#run-unit-test).
## What contributions can I make?
Contributions to Milvus fall into the following categories.
1. To report a bug or a problem with documentation, please file an [issue](https://github.com/milvus-io/milvus/issues/new) providing the details of the problem. If you believe the issue needs priority attention, please comment on the issue to notify the team.
2. To propose a new feature, please file a new feature request [issue](https://github.com/milvus-io/milvus/issues/new). Describe the intended feature and discuss the design and implementation with the team and community. Once the team agrees that the plan looks good, go ahead and implement it, following the [Contributing code].
3. To implement a feature or bug-fix for an existing outstanding issue, follow the [Contributing code]. If you need more context on a particular issue, comment on the issue to let people know.
## How can I contribute?
### Contributing code
If you have improvements to Milvus, send us your pull requests! For those just getting started, GitHub has a [how-to](https://help.github.com/en/articles/about-pull-requests).
The Milvus team members will review your pull requests, and once it is accepted, it will be given a `ready to merge` label. This means we are working on submitting your pull request to the internal repository. After the change has been submitted internally, your pull request will be merged automatically on GitHub.
### General guidelines
Before sending your pull requests for review, make sure your changes are consistent with the guidelines and follow the Milvus coding style.
- Include unit tests when you contribute new features, as they help to a) prove that your code works correctly, and b) guard against future breaking changes to lower the maintenance cost.
- Bug fixes also generally require unit tests, because the presence of bugs usually indicates insufficient test coverage.
- Keep API compatibility in mind when you change code in Milvus. Reviewers of your pull request will comment on any API compatibility issues.
- When you contribute a new feature to Milvus, the maintenance burden is (by default) transferred to the Milvus team. This means that the benefit of the contribution must be compared against the cost of maintaining the feature.
## Coding Style
## Run unit test
```shell
$ ./build.sh -u
or
$ ./build.sh --unittest
```
cpp/NOTICE.md 0 → 100644
浏览文件 @ 3c6f49f9
| Name | License |
| ------------- | ------------------------------------------------------------ |
| Apache Arrow | Apache License 2.0 |
| Boost | Boost Software License |
| BZIP2 | BSD-like license |
| FAISS | MIT |
| Gtest | BSD 3-Clause |
| LAPACK | BSD 3-Clause |
| LZ4 | [BSD 2-Clause](https://github.com/Blosc/c-blosc/blob/master/LICENSES/LZ4.txt) |
| MySQLPP | LGPL 2.1 |
| OpenBLAS | BSD 3-Clause |
| Prometheus | Apache License 2.0 |
| Snappy | [BSD](https://github.com/Blosc/c-blosc/blob/master/LICENSES/SNAPPY.txt) |
| SQLite | [Public Domain](https://www.sqlite.org/copyright.html) |
| SQLite-ORM | BSD 3-Clause |
| yaml-cpp | MIT |
| ZLIB | [zlib](http://zlib.net/zlib_license.html) |
| ZSTD | [BSD](https://github.com/facebook/zstd/blob/dev/LICENSE) |
| libunwind | MIT |
| gperftools | BSD 3-Clause |
| gRPC | Apache 2.0 |
| EASYLOGGINGPP | MIT |
cpp/README.md
浏览文件 @ 3c6f49f9
### Compilation
#### Step 1: install necessery tools
# Welcome to Milvus
centos7 :
yum install gfortran qt4 flex bison mysql-devel mysql
ubuntu16.04 :
sudo apt-get install gfortran qt4-qmake flex bison libmysqlclient-dev mysql-client
cd scripts && sudo ./requirements.sh
Firstly, welcome, and thanks for your interest in [Milvus](https://milvus.io)! No matter who you are, what you do, we greatly appreciate your contribution to help us reinvent data science with Milvus.
If `libmysqlclient_r.so` does not exist after installing MySQL Development Files, you need to create a symbolic link:
## What is Milvus
Milvus is an open source vector search engine that makes incredibly fast querying speed enhancement over current solutions of massive vector processing. Built on optimized indexing algorithm, it is compatible with major AI/ML models.
Milvus was developed by researchers and engineers in ZILLIZ, a tech startup that intends to reinvent data science, with the purpose of providing enterprises with efficient and scalable similarity search and analysis of feature vectors and unstructured data.
Milvus provides stable Python and C++ APIs, as well as RESTful API.
Keep up-to-date with newest releases and latest updates by reading [Releases](https://www.milvus-io/docs/master/releases).
- GPU-accelerated search engine
Milvus is designed for the largest scale of vector index. CPU/GPU heterogeneous computing architecture allows you to process data at a speed 1000 times faster.
- Intelligent index
With a “Decide Your Own Algorithm” approach, you can embed machine learning and advanced algorithms into Milvus without the headache of complex data engineering or migrating data between disparate systems. Milvus is built on optimized indexing algorithm based on quantization indexing, tree-based and graph indexing methods.
- Strong scalability
The data is stored and computed on a distributed architecture. This lets you scale data sizes up and down without redesigning the system.
## Architecture
![Milvus_arch](https://www.milvus-io/docs/master/assets/milvus_arch.png)
## Get started
### Install and start Milvus server
#### Use Docker
Use Docker to install Milvus is a breeze. See the [Milvus install guide](https://www.milvus-io/docs/master/userguide/install_milvus.md) for details.
#### Use source code
##### Compilation
###### Step 1 Install necessary tools
```shell
# Install tools
Centos7 :
$ yum install gfortran qt4 flex bison mysql-devel mysql
Ubuntu16.04 :
$ sudo apt-get install gfortran qt4-qmake flex bison libmysqlclient-dev mysql-client
```
Verify the existence of `libmysqlclient_r.so`:
```shell
# Verify existence
$ locate libmysqlclient_r.so
```
sudo ln -s /path/to/libmysqlclient.so /path/to/libmysqlclient_r.so
If not, you need to create a symbolic link:
```shell
# Create symbolic link
$ sudo ln -s /path/to/libmysqlclient.so /path/to/libmysqlclient_r.so
```
#### Step 2: build(output to cmake_build folder)
###### Step 2 Build
cmake_build/src/milvus_server is the server
```shell
TBD
cd [Milvus sourcecode path]/cpp/thirdparty
git clone git@192.168.1.105:megasearch/knowhere.git
cd knowhere
./build.sh -t Debug
or ./build.sh -t Release
cd [sourcecode path]/cpp/thirdparty
git clone git@192.168.1.105:megasearch/knowhere.git
cd knowhere
./build.sh -t Debug
or ./build.sh -t Release
cd [sourcecode path]/cpp
./build.sh -t Debug
or ./build.sh -t Release
```
cd [sourcecode path]/cpp
./build.sh -t Debug
or ./build.sh -t Release
When the build is completed, all the stuff that you need in order to run Milvus will be installed under `[Milvus root path]/cpp/milvus`.
If you encounter the following error when building:
If you encounter the following error message,
`protocol https not supported or disabled in libcurl`
1. Install libcurl4-openssl-dev
2. Install cmake 3.14:
please reinstall CMake with curl:
1. Install curl development files:
```shell
CentOS 7:
$ yum install curl-devel
Ubuntu 16.04:
$ sudo apt-get install libcurl4-openssl-dev
```
./bootstrap --system-curl
make
sudo make install
2. Install [CMake 3.14](https://github.com/Kitware/CMake/releases/download/v3.14.6/cmake-3.14.6.tar.gz):
```shell
$ ./bootstrap --system-curl
$ make
$ sudo make install
```
#### To build unittest:
##### Run unit test
```shell
$ ./build.sh -u
or
$ ./build.sh --unittest
```
./build.sh -u
or
./build.sh --unittest
##### Run code coverage
#### To run code coverage
```shell
CentOS 7:
$ yum install lcov
Ubuntu 16.04:
$ sudo apt-get install lcov
$ ./build.sh -u -c
```
apt-get install lcov
./build.sh -u -c
##### Launch Milvus server
### Launch server
Set config in cpp/conf/server_config.yaml
```shell
$ cd [Milvus root path]/cpp/milvus
```
Add milvus/lib to LD_LIBRARY_PATH
Add `lib/` directory to `LD_LIBRARY_PATH`
```
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/path/to/milvus/lib
$ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/path/to/milvus/lib
```
Then start Milvus server:
```
$ cd scripts
$ ./start_server.sh
```
To stop Milvus server, run:
```shell
$ ./stop_server.sh
```
To edit Milvus settings in `conf/server_config.yaml` and `conf/log_config.conf`, please read [Milvus Configuration](https://www.milvus-io/docs/master/reference/milvus_config.md).
### Try your first Milvus program
#### Run Python example code
Make sure [Python 3.4](https://www.python.org/downloads/) or higher is already installed and in use.
Install Milvus Python SDK.
```shell
# Install Milvus Python SDK
$ pip install pymilvus==0.2.0
```
Create a new file `example.py`, and add [Python example code](https://github.com/milvus-io/pymilvus/blob/branch-0.3.1/examples/AdvancedExample.py) to it.
Run the example code.
```python
# Run Milvus Python example
$ python3 example.py
```
#### Run C++ example code
```shell
# Run Milvus C++ example
$ cd [Milvus root path]/cpp/milvus/bin
$ ./sdk_simple
```
## Contribution guidelines
Contributions are welcomed and greatly appreciated. If you want to contribute to Milvus, please read the [contribution guidelines](CONTRIBUTING.md). This project adheres to the [code of conduct](CODE OF CONDUCT.md) of Milvus. By participating, you are expected to uphold this code.
We use [GitHub issues] to track issues and bugs. For general questions and discussions, please go to [Milvus Forum].
## Join the Milvus community
For public discussion of Milvus, please join our [discussion group].
## Milvus Roadmap
Please read our [roadmap](milvus-io/milvus/docs/master/roadmap.md) to learn about upcoming features.
## Resources
[Milvus official website](https://milvus.io)
[Milvus docs](https://www.milvus.io/docs/en/QuickStart/)
[Milvus blog](https://www.milvus.io/blog/)
[Milvus CSDN](https://mp.csdn.net/mdeditor/100041006#)
[Milvus roadmap](https://www.milvus-io/docs/master/roadmap.md)
[Milvus white paper]
Then launch server with config:
cd [build output path]
start_server.sh
stop_server.sh
## License
### Launch test_client(only for debug)
If you want to test remote api, you can run sdk example.
[build output path]/sdk/examples/grpcsimple/sdk_simple
[Apache 2.0 license](milvus-io/milvus/LICENSE.md)
\ No newline at end of file
cpp/cmake/DefineOptions.cmake
浏览文件 @ 3c6f49f9
......@@ -55,8 +55,6 @@ define_option_string(MILVUS_DEPENDENCY_SOURCE
define_option(MILVUS_VERBOSE_THIRDPARTY_BUILD
"Show output from ExternalProjects rather than just logging to files" ON)
define_option(MILVUS_WITH_ARROW "Build with ARROW" OFF)
define_option(MILVUS_BOOST_VENDORED "Use vendored Boost instead of existing Boost. \
Note that this requires linking Boost statically" ON)
......@@ -66,22 +64,10 @@ define_option(MILVUS_WITH_BZ2 "Build with BZ2 compression" ON)
define_option(MILVUS_WITH_EASYLOGGINGPP "Build with Easylogging++ library" ON)
define_option(MILVUS_WITH_FAISS "Build with FAISS library" OFF)
define_option(MILVUS_WITH_FAISS_GPU_VERSION "Build with FAISS GPU version" OFF)
define_option(MILVUS_WITH_LAPACK "Build with LAPACK library" OFF)
define_option(MILVUS_WITH_LZ4 "Build with lz4 compression" ON)
define_option(MILVUS_WITH_JSONCONS "Build with JSONCONS" OFF)
define_option(MILVUS_WITH_OPENBLAS "Build with OpenBLAS library" OFF)
define_option(MILVUS_WITH_PROMETHEUS "Build with PROMETHEUS library" ON)
define_option(MILVUS_WITH_ROCKSDB "Build with RocksDB library" OFF)
define_option(MILVUS_WITH_SNAPPY "Build with Snappy compression" ON)
define_option(MILVUS_WITH_SQLITE "Build with SQLite library" ON)
......@@ -94,10 +80,6 @@ define_option(MILVUS_WITH_YAMLCPP "Build with yaml-cpp library" ON)
define_option(MILVUS_WITH_ZLIB "Build with zlib compression" ON)
define_option(MILVUS_WITH_KNOWHERE "Build with Knowhere" OFF)
#define_option(MILVUS_ENABLE_PROFILING "Build with profiling" ON)
if(CMAKE_VERSION VERSION_LESS 3.7)
set(MILVUS_WITH_ZSTD_DEFAULT OFF)
else()
......@@ -106,13 +88,13 @@ else()
endif()
define_option(MILVUS_WITH_ZSTD "Build with zstd compression" ${MILVUS_WITH_ZSTD_DEFAULT})
define_option(MILVUS_WITH_AWS "Build with AWS SDK" ON)
if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
define_option(MILVUS_WITH_LIBUNWIND "Build with libunwind" ON)
define_option(MILVUS_WITH_GPERFTOOLS "Build with gperftools" ON)
endif()
define_option(MILVUS_WITH_GRPC "Build with GRPC" ON)
#----------------------------------------------------------------------
if(MSVC)
set_option_category("MSVC")
......
cpp/cmake/ThirdPartyPackages.cmake
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/coverage.sh
浏览文件 @ 3c6f49f9
......@@ -104,6 +104,8 @@ ${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \
"src/server/Server.cpp"\
"src/server/DBWrapper.cpp"\
"src/server/grpc_impl/GrpcMilvusServer.cpp"\
"src/utils/easylogging++.h"\
"src/utils/easylogging++.cc"\
# gen html report
${LCOV_GEN_CMD} "${FILE_INFO_OUTPUT_NEW}" --output-directory ${DIR_LCOV_OUTPUT}/
\ No newline at end of file
cpp/src/CMakeLists.txt
浏览文件 @ 3c6f49f9
......@@ -17,69 +17,65 @@
# under the License.
#-------------------------------------------------------------------------------
include_directories(${MILVUS_SOURCE_DIR})
include_directories(${MILVUS_ENGINE_SRC})
add_subdirectory(core)
set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE)
foreach(dir ${CORE_INCLUDE_DIRS})
foreach (dir ${CORE_INCLUDE_DIRS})
include_directories(${dir})
endforeach()
aux_source_directory(cache cache_files)
aux_source_directory(config config_files)
aux_source_directory(server server_files)
aux_source_directory(server/grpc_impl grpcserver_files)
aux_source_directory(utils utils_files)
aux_source_directory(db db_main_files)
aux_source_directory(db/engine db_engine_files)
aux_source_directory(db/insert db_insert_files)
aux_source_directory(db/meta db_meta_files)
aux_source_directory(metrics metrics_files)
aux_source_directory(wrapper/knowhere knowhere_files)
aux_source_directory(scheduler/action scheduler_action_files)
aux_source_directory(scheduler/event scheduler_event_files)
aux_source_directory(scheduler/job scheduler_job_files)
aux_source_directory(scheduler/resource scheduler_resource_files)
aux_source_directory(scheduler/task scheduler_task_files)
aux_source_directory(scheduler scheduler_root_files)
set(scheduler_srcs
endforeach ()
aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler db_scheduler_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context db_scheduler_context_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task db_scheduler_task_files)
set(db_scheduler_files
${db_scheduler_files}
${db_scheduler_context_files}
${db_scheduler_task_files}
)
set(grpc_service_files
${MILVUS_ENGINE_SRC}/grpc/gen-milvus/milvus.grpc.pb.cc
${MILVUS_ENGINE_SRC}/grpc/gen-milvus/milvus.pb.cc
${MILVUS_ENGINE_SRC}/grpc/gen-status/status.grpc.pb.cc
${MILVUS_ENGINE_SRC}/grpc/gen-status/status.pb.cc
)
aux_source_directory(${MILVUS_ENGINE_SRC}/metrics metrics_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_main_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/job scheduler_job_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_files)
set(scheduler_files
${scheduler_main_files}
${scheduler_action_files}
${scheduler_event_files}
${scheduler_job_files}
${scheduler_resource_files}
${scheduler_task_files}
${scheduler_root_files}
)
aux_source_directory(db/scheduler scheduler_files)
aux_source_directory(db/scheduler/context scheduler_context_files)
aux_source_directory(db/scheduler/task scheduler_task_files)
set(db_scheduler_files
${scheduler_files}
${scheduler_context_files}
${scheduler_task_files}
)
set(license_check_files
license/LicenseLibrary.cpp
license/LicenseCheck.cpp
)
aux_source_directory(${MILVUS_ENGINE_SRC}/server server_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl grpc_server_files)
set(license_generator_files
license/LicenseGenerator.cpp
license/LicenseLibrary.cpp
)
aux_source_directory(${MILVUS_ENGINE_SRC}/utils utils_files)
set(grpc_service_files
metrics/SystemInfo.cpp
metrics/SystemInfo.h
grpc/gen-milvus/milvus.grpc.pb.cc
grpc/gen-milvus/milvus.pb.cc
grpc/gen-status/status.grpc.pb.cc
grpc/gen-status/status.pb.cc
scheduler/Utils.h)
set(db_files
aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_files)
set(engine_files
${CMAKE_CURRENT_SOURCE_DIR}/main.cpp
${cache_files}
${db_main_files}
......@@ -88,38 +84,40 @@ set(db_files
${db_meta_files}
${db_scheduler_files}
${metrics_files}
${knowhere_files}
${utils_files}
${wrapper_files}
)
set(s3_client_files
storage/s3/S3ClientWrapper.cpp
storage/s3/S3ClientWrapper.h)
include_directories(/usr/include)
include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include")
include_directories(/usr/include/mysql)
include_directories(grpc/gen-status)
include_directories(grpc/gen-milvus)
include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-status)
include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-milvus)
set(client_grpc_lib
grpcpp_channelz
grpc++
grpc
grpc_protobuf
grpc_protoc)
grpc_protoc
)
set(third_party_libs
knowhere
sqlite
${client_grpc_lib}
yaml-cpp
set(prometheus_lib
prometheus-cpp-push
prometheus-cpp-pull
prometheus-cpp-core
)
set(boost_lib
boost_system_static
boost_filesystem_static
boost_serialization_static
)
set(third_party_libs
sqlite
${client_grpc_lib}
yaml-cpp
${prometheus_lib}
${boost_lib}
bzip2
lz4
snappy
......@@ -131,13 +129,12 @@ set(third_party_libs
${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so
cudart
)
if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
set(third_party_libs ${third_party_libs}
gperftools
libunwind)
endif()
libunwind
)
endif ()
link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
set(engine_libs
......@@ -149,22 +146,19 @@ set(engine_libs
if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
set(engine_libs
${engine_libs}
libquadmath.a
)
${engine_libs}
libquadmath.a
)
endif ()
cuda_add_library(milvus_engine STATIC ${db_files})
cuda_add_library(milvus_engine STATIC ${engine_files})
target_link_libraries(milvus_engine ${engine_libs} ${third_party_libs})
add_library(metrics STATIC ${metrics_files})
set(metrics_lib
yaml-cpp
prometheus-cpp-push
prometheus-cpp-pull
prometheus-cpp-core
${prometheus_lib}
)
target_link_libraries(metrics ${metrics_lib})
......@@ -176,21 +170,17 @@ set(server_libs
metrics
)
set(knowhere_libs
knowhere
)
add_executable(milvus_server
${config_files}
${metrics_files}
${scheduler_files}
${server_files}
${grpcserver_files}
${utils_files}
${grpc_server_files}
${grpc_service_files}
${metrics_files}
${scheduler_srcs}
${utils_files}
)
target_link_libraries(milvus_server ${server_libs} ${knowhere_libs} ${third_party_libs})
target_link_libraries(milvus_server ${server_libs} knowhere ${third_party_libs})
install(TARGETS milvus_server DESTINATION bin)
......
cpp/src/cache/DataObj.h
浏览文件 @ 3c6f49f9
......@@ -18,7 +18,7 @@
#pragma once
#include "wrapper/knowhere/vec_index.h"
#include "src/wrapper/vec_index.h"
#include <memory>
......
cpp/src/core/CMakeLists.txt
浏览文件 @ 3c6f49f9
......@@ -84,7 +84,7 @@ include(DefineOptionsCore)
include(BuildUtilsCore)
include(ThirdPartyPackagesCore)
add_subdirectory(src)
add_subdirectory(knowhere)
if (BUILD_COVERAGE STREQUAL "ON")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
......
cpp/src/core/include/knowhere/adapter/arrow.h 已删除 100644 → 0
浏览文件 @ e72b0348
#pragma once
#include <memory>
#include <knowhere/common/array.h>
namespace zilliz {
namespace knowhere {
ArrayPtr
CopyArray(const ArrayPtr &origin);
SchemaPtr
CopySchema(const SchemaPtr &origin);
} // namespace knowhere
} // namespace zilliz
cpp/src/core/include/knowhere/adapter/sptag.h 已删除 100644 → 0
浏览文件 @ e72b0348
#pragma once
#include <memory>
#include <knowhere/common/dataset.h>
#include <SPTAG/AnnService/inc/Core/VectorIndex.h>
namespace zilliz {
namespace knowhere {
std::shared_ptr<SPTAG::VectorSet>
ConvertToVectorSet(const DatasetPtr &dataset);
std::shared_ptr<SPTAG::MetadataSet>
ConvertToMetadataSet(const DatasetPtr &dataset);
std::vector<SPTAG::QueryResult>
ConvertToQueryResult(const DatasetPtr &dataset, const Config &config);
DatasetPtr
ConvertToDataset(std::vector<SPTAG::QueryResult> query_results);
} // namespace knowhere
} // namespace zilliz
cpp/src/core/include/knowhere/common/config.h 已删除 100644 → 0
浏览文件 @ e72b0348
#pragma once
#include <jsoncons/json.hpp>
namespace zilliz {
namespace knowhere {
using Config = jsoncons::json;
} // namespace knowhere
} // namespace zilliz
cpp/src/core/include/knowhere/common/device_type.h 已删除 100644 → 0
浏览文件 @ e72b0348
#pragma once
namespace zilliz {
namespace sched {
namespace master {
} // namespace master
} // namespace sched
} // namespace zilliz
cpp/src/core/include/knowhere/common/error.h 已删除 100644 → 0
浏览文件 @ e72b0348
#pragma once
#include <cstdint>
#include "zlibrary/error/error.h"
namespace zilliz {
namespace knowhere {
using Error = zilliz::lib::ErrorCode;
constexpr Error STORE_SUCCESS = zilliz::lib::SUCCESS_CODE;
constexpr Error ERROR_CODE_BASE = 0x36000;
constexpr Error ERROR_CODE_END = 0x37000;
constexpr Error
ToGlobalErrorCode(const Error error_code) {
return zilliz::lib::ToGlobalErrorCode(error_code, ERROR_CODE_BASE);
}
class Exception : public zilliz::lib::Exception {
public:
Exception(const Error error_code,
const std::string &message = nullptr)
: zilliz::lib::Exception(error_code, "KNOWHERE", message) {}
};
constexpr Error UNEXPECTED = ToGlobalErrorCode(0x001);
constexpr Error UNSUPPORTED = ToGlobalErrorCode(0x002);
constexpr Error NULL_POINTER = ToGlobalErrorCode(0x003);
constexpr Error OVERFLOW = ToGlobalErrorCode(0x004);
constexpr Error INVALID_ARGUMENT = ToGlobalErrorCode(0x005);
constexpr Error UNSUPPORTED_TYPE = ToGlobalErrorCode(0x006);
} // namespace store
} // namespace zilliz
using Error = zilliz::store::Error;
cpp/src/core/include/knowhere/common/id.h 已删除 100644 → 0
浏览文件 @ e72b0348
#pragma once
//#include "zcommon/id/id.h"
//using ID = zilliz::common::ID;
#include <stdint.h>
#include <string>
namespace zilliz {
namespace knowhere {
class ID {
public:
constexpr static int64_t kIDSize = 20;
public:
const int32_t *
data() const { return content_; }
int32_t *
mutable_data() { return content_; }
bool
IsValid() const;
std::string
ToString() const;
bool
operator==(const ID &that) const;
bool
operator<(const ID &that) const;
protected:
int32_t content_[5] = {};
};
} // namespace knowhere
} // namespace zilliz
cpp/src/core/include/knowhere/common/schema.h 已删除 100644 → 0
浏览文件 @ e72b0348
#pragma once
#include <memory>
#include "arrow/type.h"
namespace zilliz {
namespace knowhere {
using DataType = arrow::DataType;
using Field = arrow::Field;
using FieldPtr = std::shared_ptr<arrow::Field>;
using Schema = arrow::Schema;
using SchemaPtr = std::shared_ptr<Schema>;
using SchemaConstPtr = std::shared_ptr<const Schema>;
} // namespace knowhere
} // namespace zilliz
cpp/src/core/include/knowhere/common/tensor.h 已删除 100644 → 0
浏览文件 @ e72b0348
#pragma once
#include <memory>
#include "arrow/tensor.h"
namespace zilliz {
namespace knowhere {
using Tensor = arrow::Tensor;
using TensorPtr = std::shared_ptr<Tensor>;
} // namespace knowhere
} // namespace zilliz
cpp/src/core/include/knowhere/index/index.h 已删除 100644 → 0
浏览文件 @ e72b0348
#pragma once
#include <memory>
#include "knowhere/common/binary_set.h"
#include "knowhere/common/dataset.h"
#include "knowhere/index/index_type.h"
#include "knowhere/index/index_model.h"
#include "knowhere/index/preprocessor/preprocessor.h"
namespace zilliz {
namespace knowhere {
class Index {
public:
virtual BinarySet
Serialize() = 0;
virtual void
Load(const BinarySet &index_binary) = 0;
// @throw
virtual DatasetPtr
Search(const DatasetPtr &dataset, const Config &config) = 0;
public:
IndexType
idx_type() const { return idx_type_; }
void
set_idx_type(IndexType idx_type) { idx_type_ = idx_type; }
virtual void
set_preprocessor(PreprocessorPtr preprocessor) {}
virtual void
set_index_model(IndexModelPtr model) {}
private:
IndexType idx_type_;
};
using IndexPtr = std::shared_ptr<Index>;
} // namespace knowhere
} // namespace zilliz
cpp/src/core/include/knowhere/index/index_model.h 已删除 100644 → 0
浏览文件 @ e72b0348
#pragma once
#include <memory>
#include "knowhere/common/binary_set.h"
namespace zilliz {
namespace knowhere {
class IndexModel {
public:
virtual BinarySet
Serialize() = 0;
virtual void
Load(const BinarySet &binary) = 0;
};
using IndexModelPtr = std::shared_ptr<IndexModel>;
} // namespace knowhere
} // namespace zilliz
cpp/src/core/include/knowhere/index/index_type.h 已删除 100644 → 0
浏览文件 @ e72b0348
#pragma once
namespace zilliz {
namespace knowhere {
enum class IndexType {
kUnknown = 0,
kVecIdxBegin = 100,
kVecIVFFlat = kVecIdxBegin,
kVecIdxEnd,
};
} // namespace knowhere
} // namespace zilliz
cpp/src/core/include/knowhere/index/preprocessor/preprocessor.h 已删除 100644 → 0
浏览文件 @ e72b0348
#pragma once
#include <memory>
#include "knowhere/common/dataset.h"
namespace zilliz {
namespace knowhere {
class Preprocessor {
public:
virtual DatasetPtr
Preprocess(const DatasetPtr &input) = 0;
};
using PreprocessorPtr = std::shared_ptr<Preprocessor>;
} // namespace knowhere
} // namespace zilliz
cpp/src/core/include/knowhere/index/vector_index/definitions.h 已删除 100644 → 0
浏览文件 @ e72b0348
#pragma once
#include <string>
namespace zilliz {
namespace knowhere {
#define META_ROWS ("rows")
#define META_DIM ("dimension")
#define META_K ("k")
} // namespace knowhere
} // namespace zilliz
cpp/src/core/include/knowhere/index/vector_index/kdt_parameters.h 已删除 100644 → 0
浏览文件 @ e72b0348
#pragma once
#include <string>
#include <vector>
namespace zilliz {
namespace knowhere {
using KDTParameter = std::pair<std::string, std::string>;
class KDTParameterManagement {
public:
const std::vector<KDTParameter> &
GetKDTParameters();
public:
static KDTParameterManagement &
GetInstance() {
static KDTParameterManagement instance;
return instance;
}
KDTParameterManagement(const KDTParameterManagement &) = delete;
KDTParameterManagement &operator=(const KDTParameterManagement &) = delete;
private:
KDTParameterManagement();
private:
std::vector<KDTParameter> kdt_parameters_;
};
} // namespace knowhere
} // namespace zilliz
cpp/src/core/include/knowhere/index/vector_index/nsg/nsg.i 已删除 100644 → 0
浏览文件 @ e72b0348
%module nsg
%{
#define SWIG_FILE_WITH_INIT
#include <numpy/arrayobject.h>
/* Include the header in the wrapper code */
#include "nsg.h"
%}
/* Parse the header file */
%include "index.h"
cpp/src/core/include/knowhere/index/vector_index/vector_index.h 已删除 100644 → 0
浏览文件 @ e72b0348
#pragma once
#include <memory>
#include "knowhere/common/config.h"
#include "knowhere/common/dataset.h"
#include "knowhere/index/index.h"
#include "knowhere/index/preprocessor/preprocessor.h"
namespace zilliz {
namespace knowhere {
class VectorIndex;
using VectorIndexPtr = std::shared_ptr<VectorIndex>;
class VectorIndex : public Index {
public:
virtual PreprocessorPtr
BuildPreprocessor(const DatasetPtr &dataset, const Config &config) { return nullptr; }
virtual IndexModelPtr
Train(const DatasetPtr &dataset, const Config &config) { return nullptr; }
virtual void
Add(const DatasetPtr &dataset, const Config &config) = 0;
virtual void
Seal() = 0;
virtual VectorIndexPtr
Clone() = 0;
virtual int64_t
Count() = 0;
virtual int64_t
Dimension() = 0;
};
} // namespace knowhere
} // namespace zilliz
cpp/src/core/src/CMakeLists.txt → cpp/src/core/knowhere/CMakeLists.txt
浏览文件 @ 3c6f49f9
......@@ -5,7 +5,7 @@ include_directories(${TBB_DIR}/include)
include_directories(${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
link_directories(${CUDA_TOOLKIT_ROOT_DIR}/lib64)
include_directories(${CORE_SOURCE_DIR}/include)
include_directories(${CORE_SOURCE_DIR}/knowhere)
include_directories(${CORE_SOURCE_DIR}/thirdparty)
include_directories(${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService)
include_directories(${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include)
......@@ -107,7 +107,7 @@ INSTALL(FILES ${CORE_SOURCE_DIR}/thirdparty/tbb/libtbb.so
)
set(CORE_INCLUDE_DIRS
${CORE_SOURCE_DIR}/include
${CORE_SOURCE_DIR}/knowhere
${CORE_SOURCE_DIR}/thirdparty
${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService
${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include
......@@ -120,18 +120,18 @@ set(CORE_INCLUDE_DIRS
set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE)
INSTALL(DIRECTORY
${CORE_SOURCE_DIR}/include/knowhere
${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons
${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons_ext
${ARROW_INCLUDE_DIR}/arrow
${FAISS_PREFIX}/include/faiss
${OPENBLAS_INCLUDE_DIR}/
${CORE_SOURCE_DIR}/thirdparty/tbb/include/tbb
DESTINATION
include)
INSTALL(DIRECTORY
${SPTAG_SOURCE_DIR}/AnnService/inc/
DESTINATION
include/SPTAG/AnnService/inc)
#INSTALL(DIRECTORY
# ${CORE_SOURCE_DIR}/include/knowhere
# ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons
# ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons_ext
# ${ARROW_INCLUDE_DIR}/arrow
# ${FAISS_PREFIX}/include/faiss
# ${OPENBLAS_INCLUDE_DIR}/
# ${CORE_SOURCE_DIR}/thirdparty/tbb/include/tbb
# DESTINATION
# include)
#
#INSTALL(DIRECTORY
# ${SPTAG_SOURCE_DIR}/AnnService/inc/
# DESTINATION
# include/SPTAG/AnnService/inc)
cpp/src/core/src/knowhere/adapter/arrow.cpp → cpp/src/core/knowhere/knowhere/adapter/arrow.cpp
浏览文件 @ 3c6f49f9
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "knowhere/adapter/arrow.h"
#include "arrow.h"
namespace zilliz {
namespace knowhere {
......
cpp/src/core/knowhere/knowhere/adapter/arrow.h 0 → 100644
浏览文件 @ 3c6f49f9
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <memory>
#include "knowhere/common/array.h"
namespace zilliz {
namespace knowhere {
ArrayPtr
CopyArray(const ArrayPtr &origin);
SchemaPtr
CopySchema(const SchemaPtr &origin);
} // namespace knowhere
} // namespace zilliz
cpp/src/core/src/knowhere/adapter/sptag.cpp → cpp/src/core/knowhere/knowhere/adapter/sptag.cpp
浏览文件 @ 3c6f49f9
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "knowhere/index/vector_index/definitions.h"
#include "knowhere/adapter/sptag.h"
#include "knowhere/adapter/structure.h"
#include "sptag.h"
#include "structure.h"
namespace zilliz {
......
cpp/src/core/knowhere/knowhere/adapter/sptag.h 0 → 100644
浏览文件 @ 3c6f49f9
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <memory>
#include <SPTAG/AnnService/inc/Core/VectorIndex.h>
#include "knowhere/common/dataset.h"
namespace zilliz {
namespace knowhere {
std::shared_ptr<SPTAG::VectorSet>
ConvertToVectorSet(const DatasetPtr &dataset);
std::shared_ptr<SPTAG::MetadataSet>
ConvertToMetadataSet(const DatasetPtr &dataset);
std::vector<SPTAG::QueryResult>
ConvertToQueryResult(const DatasetPtr &dataset, const Config &config);
DatasetPtr
ConvertToDataset(std::vector<SPTAG::QueryResult> query_results);
} // namespace knowhere
} // namespace zilliz
cpp/src/core/src/knowhere/adapter/structure.cpp → cpp/src/core/knowhere/knowhere/adapter/structure.cpp
浏览文件 @ 3c6f49f9
......@@ -16,7 +16,7 @@
// under the License.
#include "knowhere/adapter/structure.h"
#include "structure.h"
namespace zilliz {
......
cpp/src/core/include/knowhere/adapter/structure.h → cpp/src/core/knowhere/knowhere/adapter/structure.h
浏览文件 @ 3c6f49f9
......@@ -19,7 +19,7 @@
#pragma once
#include <memory>
#include <knowhere/common/dataset.h>
#include "knowhere/common/dataset.h"
namespace zilliz {
......
cpp/src/core/include/knowhere/common/array.h → cpp/src/core/knowhere/knowhere/common/array.h
浏览文件 @ 3c6f49f9
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "arrow/array.h"
#include "knowhere/common/schema.h"
#include "schema.h"
namespace zilliz {
......
cpp/src/core/include/knowhere/common/binary_set.h → cpp/src/core/knowhere/knowhere/common/binary_set.h
浏览文件 @ 3c6f49f9
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <map>
......@@ -5,7 +23,7 @@
#include <vector>
#include <memory>
#include "knowhere/common/id.h"
#include "id.h"
namespace zilliz {
......
cpp/src/core/include/knowhere/common/buffer.h → cpp/src/core/knowhere/knowhere/common/buffer.h
浏览文件 @ 3c6f49f9
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <memory>
#include "arrow/buffer.h"
......
cpp/src/core/knowhere/knowhere/common/config.h 0 → 100644
浏览文件 @ 3c6f49f9
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <jsoncons/json.hpp>
namespace zilliz {
namespace knowhere {
using Config = jsoncons::json;
} // namespace knowhere
} // namespace zilliz
cpp/src/core/include/knowhere/common/dataset.h → cpp/src/core/knowhere/knowhere/common/dataset.h
浏览文件 @ 3c6f49f9
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <vector>
#include <memory>
#include "knowhere/common/array.h"
#include "knowhere/common/buffer.h"
#include "knowhere/common/tensor.h"
#include "knowhere/common/schema.h"
#include "knowhere/common/config.h"
#include "array.h"
#include "buffer.h"
#include "tensor.h"
#include "schema.h"
#include "config.h"
#include "knowhere/adapter/arrow.h"
......
cpp/src/core/src/knowhere/common/exception.cpp → cpp/src/core/knowhere/knowhere/common/exception.cpp
浏览文件 @ 3c6f49f9
......@@ -16,9 +16,10 @@
// under the License.
#include "knowhere/common/exception.h"
#include <cstdio>
#include "exception.h"
namespace zilliz {
namespace knowhere {
......
cpp/src/core/knowhere/knowhere/common/id.h 0 → 100644
浏览文件 @ 3c6f49f9
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
//#include "zcommon/id/id.h"
//using ID = zilliz::common::ID;
#include <stdint.h>
#include <string>
namespace zilliz {
namespace knowhere {
class ID {
public:
constexpr static int64_t kIDSize = 20;
public:
const int32_t *
data() const { return content_; }
int32_t *
mutable_data() { return content_; }
bool
IsValid() const;
std::string
ToString() const;
bool
operator==(const ID &that) const;
bool
operator<(const ID &that) const;
protected:
int32_t content_[5] = {};
};
} // namespace knowhere
} // namespace zilliz
cpp/src/core/knowhere/knowhere/common/schema.h 0 → 100644
浏览文件 @ 3c6f49f9
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <memory>
#include "arrow/type.h"
namespace zilliz {
namespace knowhere {
using DataType = arrow::DataType;
using Field = arrow::Field;
using FieldPtr = std::shared_ptr<arrow::Field>;
using Schema = arrow::Schema;
using SchemaPtr = std::shared_ptr<Schema>;
using SchemaConstPtr = std::shared_ptr<const Schema>;
} // namespace knowhere
} // namespace zilliz
cpp/src/core/knowhere/knowhere/common/tensor.h 0 → 100644
浏览文件 @ 3c6f49f9
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <memory>
#include "arrow/tensor.h"
namespace zilliz {
namespace knowhere {
using Tensor = arrow::Tensor;
using TensorPtr = std::shared_ptr<Tensor>;
} // namespace knowhere
} // namespace zilliz
cpp/src/core/src/knowhere/common/timer.cpp → cpp/src/core/knowhere/knowhere/common/timer.cpp
浏览文件 @ 3c6f49f9
......@@ -18,7 +18,7 @@
#include <iostream> // TODO(linxj): using Log instead
#include "knowhere/common/timer.h"
#include "timer.h"
namespace zilliz {
namespace knowhere {
......
cpp/src/core/knowhere/knowhere/index/index.h 0 → 100644
浏览文件 @ 3c6f49f9
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <memory>
#include "knowhere/common/binary_set.h"
#include "knowhere/common/dataset.h"
#include "index_type.h"
#include "index_model.h"
#include "knowhere/index/preprocessor/preprocessor.h"
namespace zilliz {
namespace knowhere {
class Index {
public:
virtual BinarySet
Serialize() = 0;
virtual void
Load(const BinarySet &index_binary) = 0;
// @throw
virtual DatasetPtr
Search(const DatasetPtr &dataset, const Config &config) = 0;
public:
IndexType
idx_type() const { return idx_type_; }
void
set_idx_type(IndexType idx_type) { idx_type_ = idx_type; }
virtual void
set_preprocessor(PreprocessorPtr preprocessor) {}
virtual void
set_index_model(IndexModelPtr model) {}
private:
IndexType idx_type_;
};
using IndexPtr = std::shared_ptr<Index>;
} // namespace knowhere
} // namespace zilliz
cpp/src/core/knowhere/knowhere/index/index_model.h 0 → 100644
浏览文件 @ 3c6f49f9
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <memory>
#include "knowhere/common/binary_set.h"
namespace zilliz {
namespace knowhere {
class IndexModel {
public:
virtual BinarySet
Serialize() = 0;
virtual void
Load(const BinarySet &binary) = 0;
};
using IndexModelPtr = std::shared_ptr<IndexModel>;
} // namespace knowhere
} // namespace zilliz
cpp/src/core/knowhere/knowhere/index/index_type.h 0 → 100644
浏览文件 @ 3c6f49f9
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
namespace zilliz {
namespace knowhere {
enum class IndexType {
kUnknown = 0,
kVecIdxBegin = 100,
kVecIVFFlat = kVecIdxBegin,
kVecIdxEnd,
};
} // namespace knowhere
} // namespace zilliz
cpp/src/core/knowhere/knowhere/index/preprocessor/preprocessor.h 0 → 100644
浏览文件 @ 3c6f49f9
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <memory>
#include "knowhere/common/dataset.h"
namespace zilliz {
namespace knowhere {
class Preprocessor {
public:
virtual DatasetPtr
Preprocess(const DatasetPtr &input) = 0;
};
using PreprocessorPtr = std::shared_ptr<Preprocessor>;
} // namespace knowhere
} // namespace zilliz
cpp/src/core/src/knowhere/index/vector_index/cloner.cpp → cpp/src/core/knowhere/knowhere/index/vector_index/cloner.cpp
浏览文件 @ 3c6f49f9
......@@ -17,10 +17,10 @@
#include "knowhere/common/exception.h"
#include "knowhere/index/vector_index/cloner.h"
#include "knowhere/index/vector_index/ivf.h"
#include "knowhere/index/vector_index/gpu_ivf.h"
#include "knowhere/index/vector_index/idmap.h"
#include "cloner.h"
#include "ivf.h"
#include "gpu_ivf.h"
#include "idmap.h"
namespace zilliz {
......
cpp/src/core/src/knowhere/index/vector_index/cpu_kdt_rng.cpp → cpp/src/core/knowhere/knowhere/index/vector_index/cpu_kdt_rng.cpp
浏览文件 @ 3c6f49f9
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <sstream>
#include <SPTAG/AnnService/inc/Server/QueryParser.h>
......@@ -7,10 +24,10 @@
#undef mkdir
#include "knowhere/index/vector_index/cpu_kdt_rng.h"
#include "knowhere/index/vector_index/definitions.h"
#include "cpu_kdt_rng.h"
#include "definitions.h"
//#include "knowhere/index/preprocessor/normalize.h"
#include "knowhere/index/vector_index/kdt_parameters.h"
#include "kdt_parameters.h"
#include "knowhere/adapter/sptag.h"
#include "knowhere/common/exception.h"
......
cpp/src/core/include/knowhere/index/vector_index/cpu_kdt_rng.h → cpp/src/core/knowhere/knowhere/index/vector_index/cpu_kdt_rng.h
浏览文件 @ 3c6f49f9
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <cstdint>
#include <memory>
#include "knowhere/index/vector_index/vector_index.h"
#include "vector_index.h"
#include "knowhere/index/index_model.h"
#include <SPTAG/AnnService/inc/Core/VectorIndex.h>
......
cpp/src/core/knowhere/knowhere/index/vector_index/definitions.h 0 → 100644
浏览文件 @ 3c6f49f9
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <string>
namespace zilliz {
namespace knowhere {
#define META_ROWS ("rows")
#define META_DIM ("dimension")
#define META_K ("k")
} // namespace knowhere
} // namespace zilliz
cpp/src/core/src/knowhere/index/vector_index/gpu_ivf.cpp → cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp
浏览文件 @ 3c6f49f9
......@@ -27,9 +27,9 @@
#include "knowhere/common/exception.h"
#include "knowhere/index/vector_index/cloner.h"
#include "cloner.h"
#include "knowhere/adapter/faiss_adopt.h"
#include "knowhere/index/vector_index/gpu_ivf.h"
#include "gpu_ivf.h"
#include <algorithm>
......
cpp/src/core/include/knowhere/index/vector_index/gpu_ivf.h → cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.h
浏览文件 @ 3c6f49f9
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <faiss/gpu/StandardGpuResources.h>
......
cpp/src/core/src/knowhere/index/vector_index/idmap.cpp → cpp/src/core/knowhere/knowhere/index/vector_index/idmap.cpp
浏览文件 @ 3c6f49f9
......@@ -25,7 +25,7 @@
#include "knowhere/common/exception.h"
#include "knowhere/adapter/faiss_adopt.h"
#include "knowhere/index/vector_index/idmap.h"
#include "idmap.h"
namespace zilliz {
......
cpp/src/core/src/knowhere/index/vector_index/ivf.cpp → cpp/src/core/knowhere/knowhere/index/vector_index/ivf.cpp
浏览文件 @ 3c6f49f9
......@@ -25,13 +25,12 @@
#include <faiss/index_io.h>
#include <faiss/gpu/StandardGpuResources.h>
#include <faiss/gpu/GpuAutoTune.h>
#include <knowhere/index/vector_index/ivf.h>
#include "knowhere/common/exception.h"
#include "knowhere/index/vector_index/ivf.h"
#include "ivf.h"
#include "knowhere/adapter/faiss_adopt.h"
#include "knowhere/index/vector_index/gpu_ivf.h"
#include "gpu_ivf.h"
namespace zilliz {
......
cpp/src/core/include/knowhere/index/vector_index/ivf.h → cpp/src/core/knowhere/knowhere/index/vector_index/ivf.h
浏览文件 @ 3c6f49f9
......@@ -25,7 +25,7 @@
#include <faiss/AuxIndexStructures.h>
#include <faiss/Index.h>
#include "knowhere/index/vector_index/vector_index.h"
#include "vector_index.h"
namespace zilliz {
......
cpp/src/core/src/knowhere/index/vector_index/kdt_parameters.cpp → cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.cpp
浏览文件 @ 3c6f49f9
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <mutex>
#include "knowhere/index/vector_index/kdt_parameters.h"
#include "kdt_parameters.h"
namespace zilliz {
......
cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.h 0 → 100644
浏览文件 @ 3c6f49f9
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <string>
#include <vector>
namespace zilliz {
namespace knowhere {
using KDTParameter = std::pair<std::string, std::string>;
class KDTParameterManagement {
public:
const std::vector<KDTParameter> &
GetKDTParameters();
public:
static KDTParameterManagement &
GetInstance() {
static KDTParameterManagement instance;
return instance;
}
KDTParameterManagement(const KDTParameterManagement &) = delete;
KDTParameterManagement &operator=(const KDTParameterManagement &) = delete;
private:
KDTParameterManagement();
private:
std::vector<KDTParameter> kdt_parameters_;
};
} // namespace knowhere
} // namespace zilliz
cpp/src/core/src/knowhere/index/vector_index/nsg/index_io.h → cpp/src/core/knowhere/knowhere/index/vector_index/nsg/index_io.h
浏览文件 @ 3c6f49f9
......@@ -18,7 +18,7 @@
#pragma once
#include "knowhere/index/vector_index/nsg/nsg.h"
#include "nsg.h"
namespace zilliz {
namespace knowhere {
......
cpp/src/core/src/knowhere/index/vector_index/nsg/nsg.cpp → cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg.cpp
浏览文件 @ 3c6f49f9
......@@ -22,7 +22,7 @@
#include <stack>
#include <omp.h>
#include "knowhere/index/vector_index/nsg/nsg.h"
#include "nsg.h"
#include "knowhere/common/exception.h"
#include "knowhere/common/timer.h"
#include "utils.h"
......
cpp/src/core/src/knowhere/index/vector_index/nsg/nsg_io.cpp → cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg_io.cpp
浏览文件 @ 3c6f49f9
......@@ -18,7 +18,7 @@
#include <cstring>
#include "knowhere/index/vector_index/nsg/nsg_io.h"
#include "nsg_io.h"
namespace zilliz {
......
cpp/src/core/src/knowhere/index/vector_index/nsg/utils.h → cpp/src/core/knowhere/knowhere/index/vector_index/nsg/utils.h
浏览文件 @ 3c6f49f9
......@@ -23,7 +23,7 @@
#include <faiss/AutoTune.h>
#include "knowhere/index/vector_index/nsg/nsg.h"
#include "nsg.h"
#include "knowhere/common/config.h"
......
cpp/src/core/src/knowhere/index/vector_index/nsg_index.cpp → cpp/src/core/knowhere/knowhere/index/vector_index/nsg_index.cpp
浏览文件 @ 3c6f49f9
......@@ -16,12 +16,12 @@
// under the License.
#include "knowhere/index/vector_index/nsg_index.h"
#include "nsg_index.h"
#include "knowhere/index/vector_index/nsg/nsg.h"
#include "knowhere/index/vector_index/nsg/nsg_io.h"
#include "knowhere/index/vector_index/idmap.h"
#include "knowhere/index/vector_index/ivf.h"
#include "knowhere/index/vector_index/gpu_ivf.h"
#include "idmap.h"
#include "ivf.h"
#include "gpu_ivf.h"
#include "knowhere/adapter/faiss_adopt.h"
#include "knowhere/common/exception.h"
#include "knowhere/common/timer.h"
......
cpp/src/core/include/knowhere/index/vector_index/nsg_index.h → cpp/src/core/knowhere/knowhere/index/vector_index/nsg_index.h
浏览文件 @ 3c6f49f9
......@@ -18,7 +18,7 @@
#pragma once
#include "knowhere/index/vector_index/vector_index.h"
#include "vector_index.h"
namespace zilliz {
......
cpp/src/core/knowhere/knowhere/index/vector_index/vector_index.h 0 → 100644
浏览文件 @ 3c6f49f9
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <memory>
#include "knowhere/common/config.h"
#include "knowhere/common/dataset.h"
#include "knowhere/index/index.h"
#include "knowhere/index/preprocessor/preprocessor.h"
namespace zilliz {
namespace knowhere {
class VectorIndex;
using VectorIndexPtr = std::shared_ptr<VectorIndex>;
class VectorIndex : public Index {
public:
virtual PreprocessorPtr
BuildPreprocessor(const DatasetPtr &dataset, const Config &config) { return nullptr; }
virtual IndexModelPtr
Train(const DatasetPtr &dataset, const Config &config) { return nullptr; }
virtual void
Add(const DatasetPtr &dataset, const Config &config) = 0;
virtual void
Seal() = 0;
virtual VectorIndexPtr
Clone() = 0;
virtual int64_t
Count() = 0;
virtual int64_t
Dimension() = 0;
};
} // namespace knowhere
} // namespace zilliz
cpp/src/core/test/CMakeLists.txt
浏览文件 @ 3c6f49f9
include_directories(${CORE_SOURCE_DIR}/thirdparty)
include_directories(${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService)
include_directories(${CORE_SOURCE_DIR}/include)
include_directories(${CORE_SOURCE_DIR}/knowhere)
include_directories(${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include)
include_directories(/usr/local/cuda/include)
link_directories(/usr/local/cuda/lib64)
......@@ -25,13 +25,13 @@ set(basic_libs
#<IVF-TEST>
set(ivf_srcs
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp
${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp
${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/ivf.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu_ivf.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/cloner.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/idmap.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/structure.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/common/exception.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/common/timer.cpp
utils.cpp
)
if(NOT TARGET test_ivf)
......@@ -41,13 +41,13 @@ target_link_libraries(test_ivf ${depend_libs} ${unittest_libs} ${basic_libs})
#<IDMAP-TEST>
set(idmap_srcs
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp
${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp
${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/idmap.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/ivf.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/cloner.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu_ivf.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/structure.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/common/exception.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/common/timer.cpp
utils.cpp
)
if(NOT TARGET test_idmap)
......@@ -57,14 +57,14 @@ target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs})
#<KDT-TEST>
set(kdt_srcs
${CORE_SOURCE_DIR}/src/knowhere/index/preprocessor/normalize.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/kdt_parameters.cpp
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cpu_kdt_rng.cpp
${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
${CORE_SOURCE_DIR}/src/knowhere/adapter/sptag.cpp
${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp
${CORE_SOURCE_DIR}/src/knowhere/adapter/arrow.cpp
${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/preprocessor/normalize.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/kdt_parameters.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/cpu_kdt_rng.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/structure.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/sptag.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/common/exception.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/arrow.cpp
${CORE_SOURCE_DIR}/knowhere/knowhere/common/timer.cpp
utils.cpp
)
if(NOT TARGET test_kdt)
......
cpp/src/db/DBFactory.cpp
浏览文件 @ 3c6f49f9
......@@ -33,14 +33,14 @@ namespace zilliz {
namespace milvus {
namespace engine {
Options DBFactory::BuildOption() {
DBOptions DBFactory::BuildOption() {
auto meta = MetaFactory::BuildOption();
Options options;
DBOptions options;
options.meta = meta;
return options;
}
DBPtr DBFactory::Build(const Options& options) {
DBPtr DBFactory::Build(const DBOptions& options) {
return std::make_shared<DBImpl>(options);
}
......
cpp/src/db/DBFactory.h
浏览文件 @ 3c6f49f9
......@@ -29,9 +29,9 @@ namespace engine {
class DBFactory {
public:
static Options BuildOption();
static DBOptions BuildOption();
static DBPtr Build(const Options& options);
static DBPtr Build(const DBOptions& options);
};
......
cpp/src/db/DBImpl.cpp
浏览文件 @ 3c6f49f9
......@@ -51,7 +51,7 @@ constexpr uint64_t INDEX_ACTION_INTERVAL = 1;
}
DBImpl::DBImpl(const Options& options)
DBImpl::DBImpl(const DBOptions& options)
: options_(options),
shutting_down_(true),
compact_thread_pool_(1, 1),
......@@ -77,7 +77,7 @@ Status DBImpl::Start() {
shutting_down_.store(false, std::memory_order_release);
//for distribute version, some nodes are read only
if (options_.mode != Options::MODE::READ_ONLY) {
if (options_.mode != DBOptions::MODE::READ_ONLY) {
ENGINE_LOG_TRACE << "StartTimerTasks";
bg_timer_thread_ = std::thread(&DBImpl::BackgroundTimerTask, this);
}
......@@ -98,7 +98,7 @@ Status DBImpl::Stop() {
//wait compaction/buildindex finish
bg_timer_thread_.join();
if (options_.mode != Options::MODE::READ_ONLY) {
if (options_.mode != DBOptions::MODE::READ_ONLY) {
meta_ptr_->CleanUp();
}
......@@ -684,7 +684,7 @@ void DBImpl::BackgroundCompaction(std::set<std::string> table_ids) {
meta_ptr_->Archive();
int ttl = 5*meta::M_SEC;//default: file will be deleted after 5 minutes
if (options_.mode == Options::MODE::CLUSTER) {
if (options_.mode == DBOptions::MODE::CLUSTER) {
ttl = meta::D_SEC;
}
meta_ptr_->CleanUpFilesWithTTL(ttl);
......
cpp/src/db/DBImpl.h
浏览文件 @ 3c6f49f9
......@@ -43,7 +43,7 @@ class Meta;
class DBImpl : public DB {
public:
explicit DBImpl(const Options &options);
explicit DBImpl(const DBOptions &options);
~DBImpl();
Status Start() override;
......@@ -75,28 +75,28 @@ class DBImpl : public DB {
Status DropIndex(const std::string& table_id) override;
Status Query(const std::string &table_id,
uint64_t k,
uint64_t nq,
uint64_t nprobe,
const float *vectors,
QueryResults &results) override;
uint64_t k,
uint64_t nq,
uint64_t nprobe,
const float *vectors,
QueryResults &results) override;
Status Query(const std::string &table_id,
uint64_t k,
uint64_t nq,
uint64_t nprobe,
const float *vectors,
const meta::DatesT &dates,
QueryResults &results) override;
uint64_t k,
uint64_t nq,
uint64_t nprobe,
const float *vectors,
const meta::DatesT &dates,
QueryResults &results) override;
Status Query(const std::string &table_id,
const std::vector<std::string> &file_ids,
uint64_t k,
uint64_t nq,
uint64_t nprobe,
const float *vectors,
const meta::DatesT &dates,
QueryResults &results) override;
const std::vector<std::string> &file_ids,
uint64_t k,
uint64_t nq,
uint64_t nprobe,
const float *vectors,
const meta::DatesT &dates,
QueryResults &results) override;
Status Size(uint64_t &result) override;
......@@ -129,7 +129,7 @@ class DBImpl : public DB {
Status MemSerialize();
private:
const Options options_;
const DBOptions options_;
std::atomic<bool> shutting_down_;
......
cpp/src/db/Options.cpp
浏览文件 @ 3c6f49f9
......@@ -27,9 +27,6 @@ namespace zilliz {
namespace milvus {
namespace engine {
Options::Options() {
}
ArchiveConf::ArchiveConf(const std::string& type, const std::string& criterias) {
ParseType(type);
ParseCritirias(criterias);
......
cpp/src/db/Options.h
浏览文件 @ 3c6f49f9
......@@ -58,16 +58,13 @@ struct DBMetaOptions {
ArchiveConf archive_conf = ArchiveConf("delete");
}; // DBMetaOptions
struct Options {
struct DBOptions {
typedef enum {
SINGLE,
CLUSTER,
READ_ONLY
} MODE;
Options();
uint16_t merge_trigger_number = 2;
DBMetaOptions meta;
int mode = MODE::SINGLE;
......
cpp/src/db/Utils.cpp
浏览文件 @ 3c6f49f9
......@@ -80,17 +80,17 @@ Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id
std::string db_path = options.path;
std::string table_path = db_path + TABLES_FOLDER + table_id;
auto status = server::CommonUtil::CreateDirectory(table_path);
if (status != 0) {
ENGINE_LOG_ERROR << "Create directory " << table_path << " Error";
return Status(DB_ERROR, "Failed to create table path");
if (!status.ok()) {
ENGINE_LOG_ERROR << status.message();
return status;
}
for(auto& path : options.slave_paths) {
table_path = path + TABLES_FOLDER + table_id;
status = server::CommonUtil::CreateDirectory(table_path);
if (status != 0) {
ENGINE_LOG_ERROR << "Create directory " << table_path << " Error";
return Status(DB_ERROR, "Failed to create table path");
if (!status.ok()) {
ENGINE_LOG_ERROR << status.message();
return status;
}
}
......@@ -120,9 +120,9 @@ Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema&
std::string parent_path = GetTableFileParentFolder(options, table_file);
auto status = server::CommonUtil::CreateDirectory(parent_path);
if (status != 0) {
ENGINE_LOG_ERROR << "Create directory " << parent_path << " Error";
return Status(DB_ERROR, "Failed to create partition directory");
if (!status.ok()) {
ENGINE_LOG_ERROR << status.message();
return status;
}
table_file.location_ = parent_path + "/" + table_file.file_id_;
......@@ -148,7 +148,9 @@ Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& tab
}
std::string msg = "Table file doesn't exist: " + file_path;
ENGINE_LOG_ERROR << msg;
ENGINE_LOG_ERROR << msg << " in path: " << options.path
<< " for table: " << table_file.table_id_;
return Status(DB_ERROR, msg);
}
......
cpp/src/db/engine/ExecutionEngineImpl.cpp
浏览文件 @ 3c6f49f9
......@@ -23,8 +23,8 @@
#include "utils/CommonUtil.h"
#include "utils/Exception.h"
#include "wrapper/knowhere/vec_index.h"
#include "wrapper/knowhere/vec_impl.h"
#include "src/wrapper/vec_index.h"
#include "src/wrapper/vec_impl.h"
#include "knowhere/common/exception.h"
#include <stdexcept>
......
cpp/src/db/engine/ExecutionEngineImpl.h
浏览文件 @ 3c6f49f9
......@@ -18,7 +18,7 @@
#pragma once
#include "ExecutionEngine.h"
#include "wrapper/knowhere/vec_index.h"
#include "src/wrapper/vec_index.h"
#include <memory>
#include <string>
......
cpp/src/db/insert/MemManagerImpl.h
浏览文件 @ 3c6f49f9
......@@ -38,7 +38,7 @@ class MemManagerImpl : public MemManager {
public:
using Ptr = std::shared_ptr<MemManagerImpl>;
MemManagerImpl(const meta::MetaPtr &meta, const Options &options)
MemManagerImpl(const meta::MetaPtr &meta, const DBOptions &options)
: meta_(meta), options_(options) {}
Status InsertVectors(const std::string &table_id,
......@@ -66,7 +66,7 @@ class MemManagerImpl : public MemManager {
MemIdMap mem_id_map_;
MemList immu_mem_list_;
meta::MetaPtr meta_;
Options options_;
DBOptions options_;
std::mutex mutex_;
std::mutex serialization_mtx_;
}; // NewMemManager
......
cpp/src/db/insert/MemMenagerFactory.cpp
浏览文件 @ 3c6f49f9
......@@ -31,8 +31,7 @@ namespace zilliz {
namespace milvus {
namespace engine {
MemManagerPtr MemManagerFactory::Build(const std::shared_ptr<meta::Meta>& meta,
const Options& options) {
MemManagerPtr MemManagerFactory::Build(const std::shared_ptr<meta::Meta>& meta, const DBOptions& options) {
return std::make_shared<MemManagerImpl>(meta, options);
}
......
cpp/src/db/insert/MemMenagerFactory.h
浏览文件 @ 3c6f49f9
......@@ -26,7 +26,7 @@ namespace engine {
class MemManagerFactory {
public:
static MemManagerPtr Build(const std::shared_ptr<meta::Meta> &meta, const Options &options);
static MemManagerPtr Build(const std::shared_ptr<meta::Meta> &meta, const DBOptions &options);
};
}
......
cpp/src/db/insert/MemTable.cpp
浏览文件 @ 3c6f49f9
......@@ -26,7 +26,7 @@ namespace engine {
MemTable::MemTable(const std::string &table_id,
const meta::MetaPtr &meta,
const Options &options) :
const DBOptions &options) :
table_id_(table_id),
meta_(meta),
options_(options) {
......@@ -54,7 +54,7 @@ Status MemTable::Add(VectorSourcePtr &source, IDNumbers &vector_ids) {
}
if (!status.ok()) {
std::string err_msg = "MemTable::Add failed: " + status.ToString();
std::string err_msg = "Insert failed: " + status.ToString();
ENGINE_LOG_ERROR << err_msg;
return Status(DB_ERROR, err_msg);
}
......@@ -74,7 +74,7 @@ Status MemTable::Serialize() {
for (auto mem_table_file = mem_table_file_list_.begin(); mem_table_file != mem_table_file_list_.end();) {
auto status = (*mem_table_file)->Serialize();
if (!status.ok()) {
std::string err_msg = "MemTable::Serialize failed: " + status.ToString();
std::string err_msg = "Insert data serialize failed: " + status.ToString();
ENGINE_LOG_ERROR << err_msg;
return Status(DB_ERROR, err_msg);
}
......
cpp/src/db/insert/MemTable.h
浏览文件 @ 3c6f49f9
......@@ -33,7 +33,7 @@ class MemTable {
public:
using MemTableFileList = std::vector<MemTableFilePtr>;
MemTable(const std::string &table_id, const meta::MetaPtr &meta, const Options &options);
MemTable(const std::string &table_id, const meta::MetaPtr &meta, const DBOptions &options);
Status Add(VectorSourcePtr &source, IDNumbers &vector_ids);
......@@ -56,7 +56,7 @@ class MemTable {
meta::MetaPtr meta_;
Options options_;
DBOptions options_;
std::mutex mutex_;
......
cpp/src/db/insert/MemTableFile.cpp
浏览文件 @ 3c6f49f9
......@@ -31,7 +31,7 @@ namespace engine {
MemTableFile::MemTableFile(const std::string &table_id,
const meta::MetaPtr &meta,
const Options &options) :
const DBOptions &options) :
table_id_(table_id),
meta_(meta),
options_(options) {
......@@ -67,7 +67,7 @@ Status MemTableFile::Add(const VectorSourcePtr &source, IDNumbers& vector_ids) {
std::string err_msg = "MemTableFile::Add: table_file_schema dimension = " +
std::to_string(table_file_schema_.dimension_) + ", table_id = " + table_file_schema_.table_id_;
ENGINE_LOG_ERROR << err_msg;
return Status(DB_ERROR, "not able to create table file");
return Status(DB_ERROR, "Not able to create table file");
}
size_t single_vector_mem_size = table_file_schema_.dimension_ * VECTOR_TYPE_SIZE;
......
cpp/src/db/insert/MemTableFile.h
浏览文件 @ 3c6f49f9
......@@ -31,7 +31,7 @@ namespace engine {
class MemTableFile {
public:
MemTableFile(const std::string &table_id, const meta::MetaPtr &meta, const Options &options);
MemTableFile(const std::string &table_id, const meta::MetaPtr &meta, const DBOptions &options);
Status Add(const VectorSourcePtr &source, IDNumbers& vector_ids);
......@@ -53,7 +53,7 @@ class MemTableFile {
meta::MetaPtr meta_;
Options options_;
DBOptions options_;
size_t current_mem_;
......
cpp/src/db/meta/MySQLMetaImpl.cpp
浏览文件 @ 3c6f49f9
......@@ -135,7 +135,7 @@ Status MySQLMetaImpl::Initialize() {
ENGINE_LOG_DEBUG << "MySQL connection pool: maximum pool size = " << std::to_string(maxPoolSize);
try {
if (mode_ != Options::MODE::READ_ONLY) {
if (mode_ != DBOptions::MODE::READ_ONLY) {
CleanUp();
}
......@@ -621,7 +621,7 @@ Status MySQLMetaImpl::DeleteTable(const std::string &table_id) {
} //Scoped Connection
if (mode_ == Options::MODE::CLUSTER) {
if (mode_ == DBOptions::MODE::CLUSTER) {
DeleteTableFiles(table_id);
}
......
cpp/src/main.cpp
浏览文件 @ 3c6f49f9
......@@ -109,9 +109,9 @@ main(int argc, char *argv[]) {
}
}
server::Server* server_ptr = server::Server::Instance();
server_ptr->Init(start_daemonized, pid_filename, config_filename, log_config_file);
return server_ptr->Start();
server::Server& server = server::Server::Instance();
server.Init(start_daemonized, pid_filename, config_filename, log_config_file);
return server.Start();
}
void
......
cpp/src/sdk/CMakeLists.txt
浏览文件 @ 3c6f49f9
......@@ -20,23 +20,10 @@
aux_source_directory(interface interface_files)
include_directories(/usr/include)
include_directories(include)
include_directories(/usr/local/include)
aux_source_directory(grpc grpc_client_files)
include_directories(${CMAKE_SOURCE_DIR}/src/grpc/gen-milvus)
include_directories(${CMAKE_SOURCE_DIR}/src/grpc/gen-status)
set(grpc_service_files
${CMAKE_SOURCE_DIR}/src/grpc/gen-milvus/milvus.grpc.pb.cc
${CMAKE_SOURCE_DIR}/src/grpc/gen-milvus/milvus.pb.cc
${CMAKE_SOURCE_DIR}/src/grpc/gen-status/status.grpc.pb.cc
${CMAKE_SOURCE_DIR}/src/grpc/gen-status/status.pb.cc
)
add_library(milvus_sdk STATIC
${interface_files}
${grpc_client_files}
......
cpp/src/sdk/examples/grpcsimple/CMakeLists.txt
浏览文件 @ 3c6f49f9
......@@ -20,11 +20,6 @@
aux_source_directory(src src_files)
include_directories(src)
include_directories(../../include)
link_directories(${CMAKE_BINARY_DIR})
add_executable(sdk_simple
main.cpp
${src_files}
......
cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp
浏览文件 @ 3c6f49f9
......@@ -190,7 +190,7 @@ void DoSearch(std::shared_ptr<Connection> conn,
{
TimeRecorder rc(phase_name);
Status stat = conn->Search(TABLE_NAME, record_array, query_range_array, TOP_K, 32, topk_query_result_array);
std::cout << "SearchVector function call status: " << stat.ToString() << std::endl;
std::cout << "SearchVector function call status: " << stat.message() << std::endl;
}
auto finish = std::chrono::high_resolution_clock::now();
std::cout << "SEARCHVECTOR COST: " << std::chrono::duration_cast<std::chrono::duration<double>>(finish - start).count() << "s\n";
......@@ -207,7 +207,7 @@ ClientTest::Test(const std::string& address, const std::string& port) {
{//connect server
ConnectParam param = {address, port};
Status stat = conn->Connect(param);
std::cout << "Connect function call status: " << stat.ToString() << std::endl;
std::cout << "Connect function call status: " << stat.message() << std::endl;
}
{//server version
......@@ -223,7 +223,7 @@ ClientTest::Test(const std::string& address, const std::string& port) {
{
std::vector<std::string> tables;
Status stat = conn->ShowTables(tables);
std::cout << "ShowTables function call status: " << stat.ToString() << std::endl;
std::cout << "ShowTables function call status: " << stat.message() << std::endl;
std::cout << "All tables: " << std::endl;
for(auto& table : tables) {
int64_t row_count = 0;
......@@ -236,7 +236,7 @@ ClientTest::Test(const std::string& address, const std::string& port) {
{//create table
TableSchema tb_schema = BuildTableSchema();
Status stat = conn->CreateTable(tb_schema);
std::cout << "CreateTable function call status: " << stat.ToString() << std::endl;
std::cout << "CreateTable function call status: " << stat.message() << std::endl;
PrintTableSchema(tb_schema);
bool has_table = conn->HasTable(tb_schema.table_name);
......@@ -248,7 +248,7 @@ ClientTest::Test(const std::string& address, const std::string& port) {
{//describe table
TableSchema tb_schema;
Status stat = conn->DescribeTable(TABLE_NAME, tb_schema);
std::cout << "DescribeTable function call status: " << stat.ToString() << std::endl;
std::cout << "DescribeTable function call status: " << stat.message() << std::endl;
PrintTableSchema(tb_schema);
}
......@@ -279,7 +279,7 @@ ClientTest::Test(const std::string& address, const std::string& port) {
std::cout << "InsertVector cost: " << std::chrono::duration_cast<std::chrono::duration<double>>(finish - start).count() << "s\n";
std::cout << "InsertVector function call status: " << stat.ToString() << std::endl;
std::cout << "InsertVector function call status: " << stat.message() << std::endl;
std::cout << "Returned id array count: " << record_ids.size() << std::endl;
if(search_record_array.size() < NQ) {
......@@ -305,16 +305,16 @@ ClientTest::Test(const std::string& address, const std::string& port) {
index.index_type = IndexType::gpu_ivfsq8;
index.nlist = 16384;
Status stat = conn->CreateIndex(index);
std::cout << "CreateIndex function call status: " << stat.ToString() << std::endl;
std::cout << "CreateIndex function call status: " << stat.message() << std::endl;
IndexParam index2;
stat = conn->DescribeIndex(TABLE_NAME, index2);
std::cout << "DescribeIndex function call status: " << stat.ToString() << std::endl;
std::cout << "DescribeIndex function call status: " << stat.message() << std::endl;
}
{//preload table
Status stat = conn->PreloadTable(TABLE_NAME);
std::cout << "PreloadTable function call status: " << stat.ToString() << std::endl;
std::cout << "PreloadTable function call status: " << stat.message() << std::endl;
}
{//search vectors after build index finish
......@@ -326,7 +326,7 @@ ClientTest::Test(const std::string& address, const std::string& port) {
{//delete index
Status stat = conn->DropIndex(TABLE_NAME);
std::cout << "DropIndex function call status: " << stat.ToString() << std::endl;
std::cout << "DropIndex function call status: " << stat.message() << std::endl;
int64_t row_count = 0;
stat = conn->CountTable(TABLE_NAME, row_count);
......@@ -339,12 +339,12 @@ ClientTest::Test(const std::string& address, const std::string& port) {
rg.end_value = CurrentTmDate(-3);
Status stat = conn->DeleteByRange(rg, TABLE_NAME);
std::cout << "DeleteByRange function call status: " << stat.ToString() << std::endl;
std::cout << "DeleteByRange function call status: " << stat.message() << std::endl;
}
{//delete table
Status stat = conn->DropTable(TABLE_NAME);
std::cout << "DeleteTable function call status: " << stat.ToString() << std::endl;
std::cout << "DeleteTable function call status: " << stat.message() << std::endl;
}
{//server status
......
cpp/src/sdk/include/Status.h
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/src/sdk/interface/Status.cpp
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/src/server/DBWrapper.cpp
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/src/server/DBWrapper.h
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/src/server/Server.cpp
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/src/server/Server.h
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/src/server/ServerConfig.cpp
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/src/server/ServerConfig.h
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/src/storage/IStorage.h 已删除 100644 → 0
浏览文件 @ e72b0348
此差异已折叠。
cpp/src/storage/s3/S3ClientWrapper.cpp 已删除 100644 → 0
浏览文件 @ e72b0348
此差异已折叠。
cpp/src/storage/s3/S3ClientWrapper.h 已删除 100644 → 0
浏览文件 @ e72b0348
此差异已折叠。
cpp/src/utils/BlockingQueue.inl
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/src/utils/CommonUtil.cpp
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/src/utils/CommonUtil.h
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/src/utils/LogUtil.cpp
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/src/utils/LogUtil.h
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/src/utils/SignalUtil.cpp
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/src/utils/Status.cpp
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/src/utils/Status.h
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/src/utils/StringHelpFunctions.h
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/src/utils/ValidationUtil.cpp
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/src/utils/ValidationUtil.h
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/src/wrapper/gpu/Arithmetic.h 已删除 100644 → 0
浏览文件 @ e72b0348
此差异已折叠。
cpp/src/wrapper/gpu/Topk.cu 已删除 100644 → 0
浏览文件 @ e72b0348
此差异已折叠。
cpp/src/wrapper/gpu/Topk.h 已删除 100644 → 0
浏览文件 @ e72b0348
此差异已折叠。
cpp/thirdparty/versions.txt
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/unittest/CMakeLists.txt
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/unittest/db/CMakeLists.txt
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/unittest/db/mysql_meta_test.cpp
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/unittest/db/utils.cpp
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/unittest/db/utils.h
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/unittest/metrics/CMakeLists.txt
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/unittest/metrics/utils.cpp
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/unittest/metrics/utils.h
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/unittest/server/CMakeLists.txt
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/unittest/server/cache_test.cpp
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/unittest/server/config_test.cpp
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/unittest/server/rpc_test.cpp
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/unittest/server/util_test.cpp
浏览文件 @ 3c6f49f9
此差异已折叠。
cpp/unittest/storage/CMakeLists.txt 已删除 100644 → 0
浏览文件 @ e72b0348
此差异已折叠。
cpp/unittest/storage/S3ClientTest.cpp 已删除 100644 → 0
浏览文件 @ e72b0348
此差异已折叠。
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或