diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 9a0f065809cfd10661552b3f603ec31170921a3e..0000000000000000000000000000000000000000 --- a/.gitmodules +++ /dev/null @@ -1,4 +0,0 @@ -[submodule "cpp/thirdparty/knowhere"] - path = cpp/thirdparty/knowhere - url = git@192.168.1.105:megasearch/knowhere.git - branch = branch-0.4.0 diff --git a/ci/jenkinsfile/milvus_build.groovy b/ci/jenkinsfile/milvus_build.groovy index dc735caa25507ac5a665dd24da3ce855d1920639..5c9a03871553db92cebc06c87cc290c9b80bf248 100644 --- a/ci/jenkinsfile/milvus_build.groovy +++ b/ci/jenkinsfile/milvus_build.groovy @@ -4,6 +4,10 @@ container('milvus-build-env') { dir ("milvus_engine") { try { checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + dir ("cpp/thirdparty/knowhere/knowhere") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/knowhere.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + } + dir ("cpp") { sh "git config --global user.email \"test@zilliz.com\"" sh "git config --global user.name \"test\"" diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 592cb79c874475fb5465e115225db95b1bfed9a3..e7cbc48a019bd279fc896227e2e04087d93bd429 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -3,15 +3,17 @@ Please mark all change in change log and use the ticket from JIRA. # Milvus 0.4.0 (2019-07-28) -- MS-336 - Scheduler interface -- MS-344 - Add TaskTable Test -- MS-345 - Add Node Test -- MS-346 - Add some implementation of scheduler to solve compile error ## Bug ## Improvement - MS-327 - Clean code for milvus +- MS-336 - Scheduler interface +- MS-344 - Add TaskTable Test +- MS-345 - Add Node Test +- MS-346 - Add some implementation of scheduler to solve compile error +- MS-348 - Add ResourceFactory Test +- MS-350 - Remove knowhere submodule ## New Feature - MS-343 - Implement ResourceMgr diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index dec7e39135c7287947001c32a38adcfe3c15f8d5..c6320522ece9fcb62321ab8494d37b6c78105f61 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -52,7 +52,7 @@ if(MILVUS_VERSION_MAJOR STREQUAL "" OR MILVUS_VERSION_MINOR STREQUAL "" OR MILVUS_VERSION_PATCH STREQUAL "") message(WARNING "Failed to determine Milvus version from git branch name") - set(MILVUS_VERSION "0.3.1") + set(MILVUS_VERSION "0.4.0") endif() message(STATUS "Build version = ${MILVUS_VERSION}") @@ -71,6 +71,7 @@ set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g") message("CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR}") message("CUDA_NVCC_FLAGS=${CUDA_NVCC_FLAGS}") +set(MILVUS_WITH_THRIFT "OFF") set(GPU_VERSION "ON") add_definitions("-DGPU_VERSION") diff --git a/cpp/README.md b/cpp/README.md index 50db004a9d782188b6dc39492e8ae500ef29f230..8e09d821e20d453fef0c49b49f0c166098497182 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -19,15 +19,15 @@ sudo ln -s /path/to/libmysqlclient.so /path/to/libmysqlclient_r.so cmake_build/src/milvus_server is the server -cmake_build/src/libmilvus_engine.a is the static library - - git submodule init - git submodule update + cd [sourcecode path]/cpp/thirdparty + git clone git@192.168.1.105:megasearch/knowhere.git + cd knowhere + ./build.sh -t Debug + or ./build.sh -t Release cd [sourcecode path]/cpp ./build.sh -t Debug - ./build.sh -t Release - ./build.sh -l -t Release # Build license version(only available for Release) + or ./build.sh -t Release If you encounter the following error when building: `protocol https not supported or disabled in libcurl` @@ -68,19 +68,5 @@ Then launch server with config: stop_server.sh ### Launch test_client(only for debug) -If you want to test remote api, you can build test_client. -test_client use same config file with server: - - cd [build output path]/test_client - test_client -c [sourcecode path]/cpp/conf/server_config.yaml - -### License Generate -Use get_sys_info to get system info file. - - ./get_sys_info # system.info will be generated - -Use license_generator to generate license file. - - ./license_generator -s system.info -l system.license -b 2019-05-15 -e 2019-08-14 - -Copy the license file to path assigned by license_path in server config file. \ No newline at end of file +If you want to test remote api, you can run sdk example. + [build output path]/sdk/examples/grpcsimple/sdk_simple diff --git a/cpp/build.sh b/cpp/build.sh index 42a7d714a612d3b0f33456ee0fe87acbadc1654d..21fcbcd90dca0c9eef0e7de9cf046e8b47e01548 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -11,7 +11,6 @@ PROFILING="OFF" BUILD_FAISS_WITH_MKL="OFF" USE_JFROG_CACHE="OFF" KNOWHERE_OPTS="" -MILVUS_WITH_THRIFT="ON" while getopts "p:d:t:uhlrcgmj" arg do @@ -52,9 +51,6 @@ do USE_JFROG_CACHE="ON" KNOWHERE_OPTS="${KNOWHERE_OPTS} -j" ;; - e) - MILVUS_WITH_THRIFT="ON" - ;; h) # help echo " @@ -69,7 +65,6 @@ parameter: -g: profiling(default: OFF) -m: build faiss with MKL(default: OFF) -j: use jfrog cache build directory --e: enable thrift usage: ./build.sh -t \${BUILD_TYPE} [-u] [-h] [-g] [-r] [-c] [-m] [-j] @@ -88,11 +83,8 @@ if [[ ! -d cmake_build ]]; then MAKE_CLEAN="ON" fi -# Build Knowhere -KNOWHERE_BUILD_DIR="`pwd`/thirdparty/knowhere_build" -pushd `pwd`/thirdparty/knowhere -./build.sh -t Release -p ${KNOWHERE_BUILD_DIR} ${KNOWHERE_OPTS} -popd +# Knowhere build output path +KNOWHERE_BUILD_DIR="`pwd`/thirdparty/knowhere/knowhere" cd cmake_build @@ -108,7 +100,6 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then -DMILVUS_DB_PATH=${DB_PATH} \ -DMILVUS_ENABLE_PROFILING=${PROFILING} \ -DBUILD_FAISS_WITH_MKL=${BUILD_FAISS_WITH_MKL} \ - -DMILVUS_WITH_THRIFT=${MILVUS_WITH_THRIFT} \ -DKNOWHERE_BUILD_DIR=${KNOWHERE_BUILD_DIR} \ -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \ $@ ../" diff --git a/cpp/src/scheduler/ResourceFactory.cpp b/cpp/src/scheduler/ResourceFactory.cpp index 65dd488e3ce9f1f195f5e1162e0030c8804737f2..496802cd195ed11c25c9a9648ff9ff19f4956952 100644 --- a/cpp/src/scheduler/ResourceFactory.cpp +++ b/cpp/src/scheduler/ResourceFactory.cpp @@ -14,11 +14,11 @@ namespace engine { std::shared_ptr ResourceFactory::Create(const std::string &name, const std::string &alias) { if (name == "disk") { - return std::make_shared(alias); + return std::make_shared(alias); } else if (name == "cpu") { return std::make_shared(alias); } else if (name == "gpu") { - return std::make_shared(alias); + return std::make_shared(alias); } else { return nullptr; } diff --git a/cpp/src/scheduler/resource/DiskResource.h b/cpp/src/scheduler/resource/DiskResource.h index b4ff32e75eb20f3d007738d4ba12b7472514f96b..77d2e978792aa3b0e3ed8d7c526b27994cc8d4ec 100644 --- a/cpp/src/scheduler/resource/DiskResource.h +++ b/cpp/src/scheduler/resource/DiskResource.h @@ -14,6 +14,17 @@ namespace milvus { namespace engine { class DiskResource : public Resource { +public: + explicit + DiskResource(std::string name) + : Resource(std::move(name), ResourceType::DISK) {} + +protected: + void + LoadFile(TaskPtr task) override {} + + void + Process(TaskPtr task) override {} }; } diff --git a/cpp/src/scheduler/resource/GpuResource.h b/cpp/src/scheduler/resource/GpuResource.h index be91950b78043a5a432e6bcfcf195708cbcd3191..74fae13b75aff90a590c07c2ac1547b31bb4a28b 100644 --- a/cpp/src/scheduler/resource/GpuResource.h +++ b/cpp/src/scheduler/resource/GpuResource.h @@ -14,6 +14,17 @@ namespace milvus { namespace engine { class GpuResource : public Resource { +public: + explicit + GpuResource(std::string name) + : Resource(std::move(name), ResourceType::GPU) {} + +protected: + void + LoadFile(TaskPtr task) override {} + + void + Process(TaskPtr task) override {} }; } diff --git a/cpp/thirdparty/knowhere b/cpp/thirdparty/knowhere deleted file mode 160000 index 98c56500b5b26a3502e2fea474586267ac6625f0..0000000000000000000000000000000000000000 --- a/cpp/thirdparty/knowhere +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 98c56500b5b26a3502e2fea474586267ac6625f0 diff --git a/cpp/unittest/scheduler/resource_factory_test.cpp b/cpp/unittest/scheduler/resource_factory_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..829fb46cdab2f10ba38ed0df1a0f2f04f3e39b43 --- /dev/null +++ b/cpp/unittest/scheduler/resource_factory_test.cpp @@ -0,0 +1,15 @@ +#include "scheduler/ResourceFactory.h" +#include + + +using namespace zilliz::milvus::engine; + +TEST(resource_factory_test, create) { + auto disk = ResourceFactory::Create("disk"); + auto cpu = ResourceFactory::Create("cpu"); + auto gpu = ResourceFactory::Create("gpu"); + + ASSERT_TRUE(std::dynamic_pointer_cast(disk)); + ASSERT_TRUE(std::dynamic_pointer_cast(cpu)); + ASSERT_TRUE(std::dynamic_pointer_cast(gpu)); +}