Merge remote-tracking branch 'upstream/branch-0.4.0' into branch-0.4.0

Former-commit-id: 259fd94df3c95b445ccd8ff34be60f7343b873ee

.gitmodules

-[submodule "cpp/thirdparty/knowhere"]
-	path = cpp/thirdparty/knowhere
-	url = git@192.168.1.105:megasearch/knowhere.git
-	branch = branch-0.4.0

ci/jenkinsfile/milvus_build.groovy

@@ -4,6 +4,10 @@ container('milvus-build-env') {
             dir ("milvus_engine") {
                 try {
                     checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
+                    dir ("cpp/thirdparty/knowhere/knowhere") {
+                        checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/knowhere.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
+                    }
+
                     dir ("cpp") {
                         sh "git config --global user.email \"test@zilliz.com\""
                         sh "git config --global user.name \"test\""

cpp/CHANGELOG.md

@@ -3,15 +3,17 @@
 Please mark all change in change log and use the ticket from JIRA.
 
 # Milvus 0.4.0 (2019-07-28)
-- MS-336 - Scheduler interface
-- MS-344 - Add TaskTable Test
-- MS-345 - Add Node Test
-- MS-346 - Add some implementation of scheduler to solve compile error
 
 ## Bug
 
 ## Improvement
 - MS-327 - Clean code for milvus
+- MS-336 - Scheduler interface
+- MS-344 - Add TaskTable Test
+- MS-345 - Add Node Test
+- MS-346 - Add some implementation of scheduler to solve compile error
+- MS-348 - Add ResourceFactory Test
+- MS-350 - Remove knowhere submodule
 
 ## New Feature
 - MS-343 - Implement ResourceMgr

cpp/CMakeLists.txt

@@ -52,7 +52,7 @@ if(MILVUS_VERSION_MAJOR STREQUAL ""
         OR MILVUS_VERSION_MINOR STREQUAL ""
         OR MILVUS_VERSION_PATCH STREQUAL "")
     message(WARNING "Failed to determine Milvus version from git branch name")
-    set(MILVUS_VERSION "0.3.1")
+    set(MILVUS_VERSION "0.4.0")
 endif()
 
 message(STATUS "Build version = ${MILVUS_VERSION}")
@@ -71,6 +71,7 @@ set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g")
 message("CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR}")
 message("CUDA_NVCC_FLAGS=${CUDA_NVCC_FLAGS}")
 
+set(MILVUS_WITH_THRIFT "OFF")
 set(GPU_VERSION "ON")
 add_definitions("-DGPU_VERSION")
 

cpp/README.md

@@ -19,15 +19,15 @@ sudo ln -s /path/to/libmysqlclient.so /path/to/libmysqlclient_r.so
 
 cmake_build/src/milvus_server is the server
 
-cmake_build/src/libmilvus_engine.a is the static library
-
-    git submodule init
-    git submodule update
+    cd [sourcecode path]/cpp/thirdparty
+    git clone git@192.168.1.105:megasearch/knowhere.git
+    cd knowhere
+    ./build.sh -t Debug
+    or ./build.sh -t Release
 
     cd [sourcecode path]/cpp
     ./build.sh -t Debug
-    ./build.sh -t Release
-    ./build.sh -l -t Release # Build license version(only available for Release)
+    or ./build.sh -t Release
 
 If you encounter the following error when building:
 `protocol https not supported or disabled in libcurl`
@@ -68,19 +68,5 @@ Then launch server with config:
     stop_server.sh
 
 ### Launch test_client(only for debug)
-If you want to test remote api, you can build test_client.
-test_client use same config file with server:
-
-    cd [build output path]/test_client
-    test_client -c [sourcecode path]/cpp/conf/server_config.yaml
-
-### License Generate
-Use get_sys_info to get system info file.
-
-    ./get_sys_info                  # system.info will be generated
-
-Use license_generator to generate license file.
-
-    ./license_generator -s system.info -l system.license -b 2019-05-15 -e 2019-08-14                 
-
-Copy the license file to path assigned by license_path in server config file.
\ No newline at end of file
+If you want to test remote api, you can run sdk example.
+    [build output path]/sdk/examples/grpcsimple/sdk_simple

cpp/build.sh

@@ -11,7 +11,6 @@ PROFILING="OFF"
 BUILD_FAISS_WITH_MKL="OFF"
 USE_JFROG_CACHE="OFF"
 KNOWHERE_OPTS=""
-MILVUS_WITH_THRIFT="ON"
 
 while getopts "p:d:t:uhlrcgmj" arg
 do
@@ -52,9 +51,6 @@ do
                 USE_JFROG_CACHE="ON"
                 KNOWHERE_OPTS="${KNOWHERE_OPTS} -j"
                 ;;
-             e)
-                MILVUS_WITH_THRIFT="ON"
-                ;;
              h) # help
                 echo "
 
@@ -69,7 +65,6 @@ parameter:
 -g: profiling(default: OFF)
 -m: build faiss with MKL(default: OFF)
 -j: use jfrog cache build directory
--e: enable thrift
 
 usage:
 ./build.sh -t \${BUILD_TYPE} [-u] [-h] [-g] [-r] [-c] [-m] [-j]
@@ -88,11 +83,8 @@ if [[ ! -d cmake_build ]]; then
 	MAKE_CLEAN="ON"
 fi
 
-# Build Knowhere
-KNOWHERE_BUILD_DIR="`pwd`/thirdparty/knowhere_build"
-pushd `pwd`/thirdparty/knowhere
-./build.sh -t Release -p ${KNOWHERE_BUILD_DIR} ${KNOWHERE_OPTS}
-popd
+# Knowhere build output path
+KNOWHERE_BUILD_DIR="`pwd`/thirdparty/knowhere/knowhere"
 
 cd cmake_build
 
@@ -108,7 +100,6 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then
     -DMILVUS_DB_PATH=${DB_PATH} \
     -DMILVUS_ENABLE_PROFILING=${PROFILING} \
     -DBUILD_FAISS_WITH_MKL=${BUILD_FAISS_WITH_MKL} \
-    -DMILVUS_WITH_THRIFT=${MILVUS_WITH_THRIFT} \
     -DKNOWHERE_BUILD_DIR=${KNOWHERE_BUILD_DIR} \
     -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
     $@ ../"

cpp/src/scheduler/ResourceFactory.cpp

@@ -14,11 +14,11 @@ namespace engine {
 std::shared_ptr<Resource>
 ResourceFactory::Create(const std::string &name, const std::string &alias) {
     if (name == "disk") {
-        return std::make_shared<CpuResource>(alias);
+        return std::make_shared<DiskResource>(alias);
     } else if (name == "cpu") {
         return std::make_shared<CpuResource>(alias);
     } else if (name == "gpu") {
-        return std::make_shared<CpuResource>(alias);
+        return std::make_shared<GpuResource>(alias);
     } else {
         return nullptr;
     }

cpp/src/scheduler/resource/DiskResource.h

@@ -14,6 +14,17 @@ namespace milvus {
 namespace engine {
 
 class DiskResource : public Resource {
+public:
+    explicit
+    DiskResource(std::string name)
+        : Resource(std::move(name), ResourceType::DISK) {}
+
+protected:
+    void
+    LoadFile(TaskPtr task) override {}
+
+    void
+    Process(TaskPtr task) override {}
 };
 
 }

cpp/src/scheduler/resource/GpuResource.h

@@ -14,6 +14,17 @@ namespace milvus {
 namespace engine {
 
 class GpuResource : public Resource {
+public:
+    explicit
+    GpuResource(std::string name)
+        : Resource(std::move(name), ResourceType::GPU) {}
+
+protected:
+    void
+    LoadFile(TaskPtr task) override {}
+
+    void
+    Process(TaskPtr task) override {}
 };
 
 }

knowhere @ 98c56500

-Subproject commit 98c56500b5b26a3502e2fea474586267ac6625f0

cpp/unittest/scheduler/resource_factory_test.cpp

+#include "scheduler/ResourceFactory.h"
+#include <gtest/gtest.h>
+
+
+using namespace zilliz::milvus::engine;
+
+TEST(resource_factory_test, create) {
+    auto disk = ResourceFactory::Create("disk");
+    auto cpu = ResourceFactory::Create("cpu");
+    auto gpu = ResourceFactory::Create("gpu");
+
+    ASSERT_TRUE(std::dynamic_pointer_cast<DiskResource>(disk));
+    ASSERT_TRUE(std::dynamic_pointer_cast<CpuResource>(cpu));
+    ASSERT_TRUE(std::dynamic_pointer_cast<GpuResource>(gpu));
+}