Merge branch 'branch-0.5.0' into refactor_knowhere

Former-commit-id: 005c77538d7c2d0b52015a748a04da6d171d0f3d

.clang-format

+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+BasedOnStyle:  Google
+DerivePointerAlignment: false
+ColumnLimit: 120

.clang-tidy

+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+Checks:          'clang-diagnostic-*,clang-analyzer-*,-clang-analyzer-alpha*,google-*,modernize-*,readability-*'
+# produce HeaderFilterRegex from cpp/build-support/lint_exclusions.txt with:
+# echo -n '^('; sed -e 's/*/\.*/g' cpp/build-support/lint_exclusions.txt | tr '\n' '|'; echo ')$'
+HeaderFilterRegex: '^(.*cmake-build-debug.*|.*cmake-build-release.*|.*cmake_build.*|.*src/thirdparty.*|.*src/core/thirdparty.*|.*src/grpc.*|)$'
+AnalyzeTemporaryDtors: true
+CheckOptions:
+  - key:             google-readability-braces-around-statements.ShortStatementLines
+    value:           '1'
+  - key:             google-readability-function-size.StatementThreshold
+    value:           '800'
+  - key:             google-readability-namespace-comments.ShortNamespaceLines
+    value:           '10'
+  - key:             google-readability-namespace-comments.SpacesBeforeComments
+    value:           '2'

.clang-tidy-ignore

+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ipc-adapter-test.cc
+# memory-pool-test.cc

ci/jenkinsfile/cluster_deploy2dev.groovy

@@ -5,7 +5,7 @@ try {
     dir ("milvus-helm") {
         checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
         dir ("milvus/milvus-cluster") {
-            sh "helm install --wait --timeout 300 --set roServers.image.tag=${DOCKER_VERSION} --set woServers.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/values.yaml --name ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster --namespace milvus-cluster --version 0.4.0 . "
+            sh "helm install --wait --timeout 300 --set roServers.image.tag=${DOCKER_VERSION} --set woServers.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/values.yaml --name ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster --namespace milvus-cluster --version 0.5.0 . "
         }
     }
     /*

ci/jenkinsfile/deploy2dev.groovy

@@ -5,7 +5,7 @@ try {
     dir ("milvus-helm") {
         checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
         dir ("milvus/milvus-gpu") {
-            sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.4.0 ."
+            sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.5.0 ."
         }
     }
 } catch (exc) {

ci/jenkinsfile/deploy2staging.groovy

@@ -5,7 +5,7 @@ try {
     dir ("milvus-helm") {
         checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
         dir ("milvus/milvus-gpu") {
-            sh "helm install --wait --timeout 300 --set engine.image.repository=\"zilliz.azurecr.cn/milvus/engine\" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.4.0 ."
+            sh "helm install --wait --timeout 300 --set engine.image.repository=\"zilliz.azurecr.cn/milvus/engine\" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.5.0 ."
         }
     }
 } catch (exc) {

ci/jenkinsfile/dev_test.groovy

@@ -3,7 +3,7 @@ timeout(time: 30, unit: 'MINUTES') {
         dir ("${PROJECT_NAME}_test") {
             checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
             sh 'python3 -m pip install -r requirements.txt'
-            sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local"
+            sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local"
         }
 
         // mysql database backend test
@@ -16,7 +16,7 @@ timeout(time: 30, unit: 'MINUTES') {
         }
         dir ("milvus-helm") {
             dir ("milvus/milvus-gpu") {
-                sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.4.0 ."
+                sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.5.0 ."
             }
         }
         dir ("${PROJECT_NAME}_test") {

ci/jenkinsfile/milvus_build.groovy

@@ -5,17 +5,12 @@ container('milvus-build-env') {
                 try {
                     checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
 
-                    /*
-                    dir ("cpp/thirdparty/knowhere") {
-                        checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/knowhere.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
-                        sh "./build.sh -t ${params.BUILD_TYPE} -p ${knowhere_build_dir} -j"
-                    }
-                    */
-
                     dir ("cpp") {
                         sh "git config --global user.email \"test@zilliz.com\""
                         sh "git config --global user.name \"test\""
-                        sh "./build.sh -t ${params.BUILD_TYPE} -j -u -c"
+                        withCredentials([usernamePassword(credentialsId: "${params.JFROG_USER}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
+                            sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -j -u -c"
+                        }
                     }
                 } catch (exc) {
                     updateGitlabCommitStatus name: 'Build Engine', state: 'failed'

ci/jenkinsfile/milvus_build_no_ut.groovy

@@ -5,17 +5,12 @@ container('milvus-build-env') {
                 try {
                     checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
 
-                    /*
-                    dir ("cpp/thirdparty/knowhere") {
-                        checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/knowhere.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
-                        sh "./build.sh -t ${params.BUILD_TYPE} -p ${knowhere_build_dir} -j"
-                    }
-                    */
-
                     dir ("cpp") {
                         sh "git config --global user.email \"test@zilliz.com\""
                         sh "git config --global user.name \"test\""
-                        sh "./build.sh -t ${params.BUILD_TYPE} -j"
+                        withCredentials([usernamePassword(credentialsId: "${params.JFROG_USER}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
+                            sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -j"
+                        }
                     }
                 } catch (exc) {
                     updateGitlabCommitStatus name: 'Build Engine', state: 'failed'

ci/jenkinsfile/staging_test.groovy

@@ -17,7 +17,7 @@ timeout(time: 40, unit: 'MINUTES') {
         }
         dir ("milvus-helm") {
             dir ("milvus/milvus-gpu") {
-                sh "helm install --wait --timeout 300 --set engine.image.repository=\"zilliz.azurecr.cn/milvus/engine\" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.4.0 ."
+                sh "helm install --wait --timeout 300 --set engine.image.repository=\"zilliz.azurecr.cn/milvus/engine\" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.5.0 ."
             }
         }
         dir ("${PROJECT_NAME}_test") {

cmake-format.py

+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# cmake-format configuration file
+# Use run-cmake-format.py to reformat all cmake files in the source tree
+
+# How wide to allow formatted cmake files
+line_width = 90
+
+# How many spaces to tab for indent
+tab_size = 2
+
+# If arglists are longer than this, break them always
+max_subargs_per_line = 4
+
+# If true, separate flow control names from their parentheses with a space
+separate_ctrl_name_with_space = False
+
+# If true, separate function names from parentheses with a space
+separate_fn_name_with_space = False
+
+# If a statement is wrapped to more than one line, than dangle the closing
+# parenthesis on it's own line
+dangle_parens = False
+
+# What style line endings to use in the output.
+line_ending = 'unix'
+
+# Format command names consistently as 'lower' or 'upper' case
+command_case = 'lower'
+
+# Format keywords consistently as 'lower' or 'upper' case
+keyword_case = 'unchanged'
+
+# enable comment markup parsing and reflow
+enable_markup = False
+
+# If comment markup is enabled, don't reflow the first comment block in
+# eachlistfile. Use this to preserve formatting of your
+# copyright/licensestatements.
+first_comment_is_literal = False
+
+# If comment markup is enabled, don't reflow any comment block which matchesthis
+# (regex) pattern. Default is `None` (disabled).
+literal_comment_pattern = None

cpp/CHANGELOG.md

@@ -18,7 +18,9 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-562 - Add JobMgr and TaskCreator in Scheduler
 - MS-566 - Refactor cmake
 - MS-555 - Remove old scheduler
-- MS-578 - Makesure milvus5.0 don't crack 0.3.1 data
+- MS-574 - Milvus configuration refactor
+- MS-578 - Make sure milvus5.0 don't crack 0.3.1 data
+- MS-585 - Update namespace in scheduler
 
 ## New Feature
 
@@ -28,6 +30,8 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-567 - Add NOTICE.md
 - MS-569 - Complete the NOTICE.md
 - MS-575 - Add Clang-format & Clang-tidy & Cpplint
+- MS-586 - Remove BUILD_FAISS_WITH_MKL option
+- MS-590 - Refine cmake code to support cpplint
 
 # Milvus 0.4.0 (2019-09-12)
 
@@ -62,6 +66,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-510 - unittest out of memory and crashed
 - MS-507 - Dataset 10m-512, index type sq8，performance in-normal when set CPU_CACHE to 16 or 64
 - MS-543 - SearchTask fail without exception
+- MS-582 - grafana displays changes frequently
 
 ## Improvement
 - MS-327 - Clean code for milvus
@@ -147,6 +152,8 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-539 - Remove old task code
 - MS-546 - Add simple mode resource_config
 - MS-570 - Add prometheus docker-compose file
+- MS-576 - Scheduler refactor
+- MS-592 - Change showtables stream transport to unary
 
 ## New Feature
 - MS-343 - Implement ResourceMgr

cpp/CMakeLists.txt

@@ -43,15 +43,7 @@ endif()
 set(MILVUS_VERSION "${GIT_BRANCH_NAME}")
 string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]" MILVUS_VERSION "${MILVUS_VERSION}")
 
-set(CLANG_FORMAT_VERSION "6.0")
 find_package(ClangTools)
-if("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1" OR CLANG_TIDY_FOUND)
-  # Generate a Clang compile_commands.json "compilation database" file for use
-  # with various development tools, such as Vim's YouCompleteMe plugin.
-  # See http://clang.llvm.org/docs/JSONCompilationDatabase.html
-  set(CMAKE_EXPORT_COMPILE_COMMANDS 1)
-endif()
-
 set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support")
 
 if(CMAKE_BUILD_TYPE STREQUAL "Release")
@@ -64,6 +56,9 @@ message(STATUS "Build type = ${BUILD_TYPE}")
 project(milvus VERSION "${MILVUS_VERSION}")
 project(milvus_engine LANGUAGES CUDA CXX)
 
+unset(CMAKE_EXPORT_COMPILE_COMMANDS CACHE)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
 set(MILVUS_VERSION_MAJOR "${milvus_VERSION_MAJOR}")
 set(MILVUS_VERSION_MINOR "${milvus_VERSION_MINOR}")
 set(MILVUS_VERSION_PATCH "${milvus_VERSION_PATCH}")
@@ -72,7 +67,7 @@ if(MILVUS_VERSION_MAJOR STREQUAL ""
         OR MILVUS_VERSION_MINOR STREQUAL ""
         OR MILVUS_VERSION_PATCH STREQUAL "")
     message(WARNING "Failed to determine Milvus version from git branch name")
-    set(MILVUS_VERSION "0.4.0")
+    set(MILVUS_VERSION "0.5.0")
 endif()
 
 message(STATUS "Build version = ${MILVUS_VERSION}")
@@ -180,12 +175,29 @@ if(NOT LINT_EXCLUSIONS_FILE)
   set(LINT_EXCLUSIONS_FILE ${BUILD_SUPPORT_DIR}/lint_exclusions.txt)
 endif()
 
+find_program(CPPLINT_BIN NAMES cpplint cpplint.py HINTS ${BUILD_SUPPORT_DIR})
+message(STATUS "Found cpplint executable at ${CPPLINT_BIN}")
+
+#
+# "make lint" targets
+#
+add_custom_target(lint
+                  ${PYTHON_EXECUTABLE}
+                  ${BUILD_SUPPORT_DIR}/run_cpplint.py
+                  --cpplint_binary
+                  ${CPPLINT_BIN}
+                  --exclude_globs
+                  ${LINT_EXCLUSIONS_FILE}
+                  --source_dir
+                  ${CMAKE_CURRENT_SOURCE_DIR}/src
+                  ${MILVUS_LINT_QUIET})
+
 #
-# "make format" and "make check-format" targets
+# "make clang-format" and "make check-clang-format" targets
 #
 if(${CLANG_FORMAT_FOUND})
   # runs clang format and updates files in place.
-  add_custom_target(format
+  add_custom_target(clang-format
                     ${PYTHON_EXECUTABLE}
                     ${BUILD_SUPPORT_DIR}/run_clang_format.py
                     --clang_format_binary
@@ -198,7 +210,7 @@ if(${CLANG_FORMAT_FOUND})
                     ${MILVUS_LINT_QUIET})
 
   # runs clang format and exits with a non-zero exit code if any files need to be reformatted
-  add_custom_target(check-format
+  add_custom_target(check-clang-format
                     ${PYTHON_EXECUTABLE}
                     ${BUILD_SUPPORT_DIR}/run_clang_format.py
                     --clang_format_binary

cpp/CONTRIBUTING.md

@@ -4,7 +4,7 @@ First of all, thanks for taking the time to contribute to Milvus! It's people li
 
 The following are a set of guidelines for contributing to Milvus. Following these guidelines helps contributing to this project easy and transparent. These are mostly guideline, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request.
 
-As for everything else in the project, the contributions to Milvus are governed by our [Code of Conduct](http://hood.ie/code-of-conduct/).
+As for everything else in the project, the contributions to Milvus are governed by our [Code of Conduct](CODE OF CONDUCT.md).
 
 TOC
 
@@ -36,21 +36,30 @@ The Milvus team members will review your pull requests, and once it is accepted,
 
 Before sending your pull requests for review, make sure your changes are consistent with the guidelines and follow the Milvus coding style.
 
-- Include unit tests when you contribute new features, as they help to a) prove that your code works correctly, and b) guard against future breaking changes to lower the maintenance cost.
-- Bug fixes also generally require unit tests, because the presence of bugs usually indicates insufficient test coverage.
+- Include unit tests when you contribute new features, as they help to prove that your code works correctly, and also guard against future breaking changes to lower the maintenance cost.
+- Bug fixes also require unit tests, because the presence of bugs usually indicates insufficient test coverage.
 - Keep API compatibility in mind when you change code in Milvus. Reviewers of your pull request will comment on any API compatibility issues.
 - When you contribute a new feature to Milvus, the maintenance burden is (by default) transferred to the Milvus team. This means that the benefit of the contribution must be compared against the cost of maintaining the feature.
 
 
 ## Coding Style
+The coding style used in Milvus generally follow [Google C++ Style Guide](https://google.github.io/styleguide/cppguide.html).
+And we made the following changes based on the guide:
+
+- 4 spaces for indentation
+- Adopt .cpp file extension instead of .cc extension
+- 120-character line length
+- Camel-Cased file names
 
 
 ## Run unit test
 
+We use Google Test framework for test running.
+To run unit test for Milvus under C++, please use the following command:
+
 ```shell
+# Run unit test for Milvus
 $ ./build.sh -u
-or
-$ ./build.sh --unittest
 ```
 
 

cpp/README.md

+- [Slack Community](https://join.slack.com/t/milvusio/shared_invite/enQtNzY1OTQ0NDI3NjMzLWNmYmM1NmNjOTQ5MGI5NDhhYmRhMGU5M2NhNzhhMDMzY2MzNDdlYjM5ODQ5MmE3ODFlYzU3YjJkNmVlNDQ2ZTk)
+- [Blog](https://www.milvus.io/blog/)
+
 # Welcome to Milvus
 
 Firstly, welcome, and thanks for your interest in [Milvus](https://milvus.io)! No matter who you are, what you do, we greatly appreciate your contribution to help us reinvent data science with Milvus.
@@ -8,9 +11,9 @@ Milvus is an open source vector search engine that supports similarity search of
 
 Milvus was developed by ZILLIZ, a tech startup that intends to reinvent data science, with the purpose of providing enterprises with efficient and scalable similarity search and analysis of feature vectors and unstructured data. 
 
-Milvus provides stable Python and C++ APIs, as well as RESTful API.
+Milvus provides stable Python, C++ and Java APIs.
 
-Keep up-to-date with newest releases and latest updates by reading [Releases](https://www.milvus-io/docs/master/releases).
+Keep up-to-date with newest releases and latest updates by reading Milvus [release notes](https://milvus.io/docs/en/Releases/v0.4.0/).
 
 - GPU-accelerated search engine
 
@@ -25,7 +28,7 @@ Keep up-to-date with newest releases and latest updates by reading [Releases](ht
   The data is stored and computed on a distributed architecture. This lets you scale data sizes up and down without redesigning the system.
 
 ## Architecture
-![Milvus_arch](https://www.milvus-io/docs/master/assets/milvus_arch.png)
+![Milvus_arch](https://milvus.io/docs/assets/milvus_arch.png)
 
 ## Get started
 
@@ -33,7 +36,7 @@ Keep up-to-date with newest releases and latest updates by reading [Releases](ht
 
 #### Use Docker
 
-Use Docker to install Milvus is a breeze. See the [Milvus install guide](https://www.milvus-io/docs/master/userguide/install_milvus.md) for details.
+Use Docker to install Milvus is a breeze. See the [Milvus install guide](https://milvus.io/docs/en/userguide/install_milvus/) for details.
 
 #### Use source code
 
@@ -44,10 +47,12 @@ Use Docker to install Milvus is a breeze. See the [Milvus install guide](https:/
 ```shell
 # Install tools
 Centos7 : 
-$ yum install gfortran qt4 flex bison mysql-devel mysql
+$ yum install gfortran qt4 flex bison 
+$ yum install mysql-devel mysql
     
 Ubuntu16.04 : 
-$ sudo apt-get install gfortran qt4-qmake flex bison libmysqlclient-dev mysql-client
+$ sudo apt-get install gfortran qt4-qmake flex bison 
+$ sudo apt-get install libmysqlclient-dev mysql-client
        
 ```
 
@@ -96,12 +101,21 @@ please reinstall CMake with curl:
    $ sudo make install
    ```
 
+##### code format and linting
+
+```shell
+CentOS 7:   
+$ yum install clang
+Ubuntu 16.04: 
+$ sudo apt-get install clang-format clang-tidy
+    
+$ ./build.sh -l
+```
+
 ##### Run unit test
 
 ```shell
 $ ./build.sh -u
-or
-$ ./build.sh --unittest
 ```
 
 ##### Run code coverage
@@ -174,17 +188,17 @@ $ python3 example.py
 
 ## Contribution guidelines
 
-Contributions are welcomed and greatly appreciated. If you want to contribute to Milvus, please read the [contribution guidelines](CONTRIBUTING.md). This project adheres to the [code of conduct](CODE OF CONDUCT.md) of Milvus. By participating, you are expected to uphold this code.
+Contributions are welcomed and greatly appreciated. If you want to contribute to Milvus, please read our [contribution guidelines](CONTRIBUTING.md). This project adheres to the [code of conduct](CODE OF CONDUCT.md) of Milvus. By participating, you are expected to uphold this code.
 
-We use [GitHub issues](https://github.com/milvus-io/milvus/issues) to track issues and bugs. For general questions and discussions, please go to [Milvus Forum]. 
+We use [GitHub issues](https://github.com/milvus-io/milvus/issues) to track issues and bugs. For general questions and public discussions, please join our community.
 
 ## Join the Milvus community
 
-For public discussion of Milvus, please join our [discussion group](milvusio.slack.com). 
+To connect with other users and contributors, welcome to join our [slack channel](https://join.slack.com/t/milvusio/shared_invite/enQtNzY1OTQ0NDI3NjMzLWNmYmM1NmNjOTQ5MGI5NDhhYmRhMGU5M2NhNzhhMDMzY2MzNDdlYjM5ODQ5MmE3ODFlYzU3YjJkNmVlNDQ2ZTk). 
 
 ## Milvus Roadmap
 
-Please read our [roadmap](milvus-io/milvus/docs/master/roadmap.md) to learn about upcoming features.
+Please read our [roadmap](https://milvus.io/docs/en/roadmap/) to learn about upcoming features.
 
 ## Resources
 
@@ -196,7 +210,7 @@ Please read our [roadmap](milvus-io/milvus/docs/master/roadmap.md) to learn abou
 
 [Milvus CSDN](https://mp.csdn.net/mdeditor/100041006#)
 
-[Milvus roadmap](https://www.milvus-io/docs/master/roadmap.md)
+[Milvus roadmap](https://milvus.io/docs/en/roadmap/)
 
 
 ## License

cpp/build-support/lint_exclusions.txt

+*cmake-build-debug*
+*cmake-build-release*
+*cmake_build*
+*src/thirdparty*
+*src/core/thirdparty*
+*src/grpc*
+*easylogging++*
\ No newline at end of file

cpp/build-support/run_cpplint.py

+#!/usr/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import print_function
+import lintutils
+from subprocess import PIPE, STDOUT
+import argparse
+import multiprocessing as mp
+import sys
+import platform
+from functools import partial
+
+
+# NOTE(wesm):
+#
+# * readability/casting is disabled as it aggressively warns about functions
+#   with names like "int32", so "int32(x)", where int32 is a function name,
+#   warns with
+_filters = '''
+-whitespace/comments
+-readability/casting
+-readability/todo
+-readability/alt_tokens
+-build/header_guard
+-build/c++11
+-runtime/references
+-build/include_order
+'''.split()
+
+
+def _get_chunk_key(filenames):
+    # lists are not hashable so key on the first filename in a chunk
+    return filenames[0]
+
+
+def _check_some_files(completed_processes, filenames):
+    # cpplint outputs complaints in '/path:line_number: complaint' format,
+    # so we can scan its output to get a list of files to fix
+    result = completed_processes[_get_chunk_key(filenames)]
+    return lintutils.stdout_pathcolonline(result, filenames)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Runs cpplint on all of the source files.")
+    parser.add_argument("--cpplint_binary",
+                        required=True,
+                        help="Path to the cpplint binary")
+    parser.add_argument("--exclude_globs",
+                        help="Filename containing globs for files "
+                        "that should be excluded from the checks")
+    parser.add_argument("--source_dir",
+                        required=True,
+                        help="Root directory of the source code")
+    parser.add_argument("--quiet", default=False,
+                        action="store_true",
+                        help="If specified, only print errors")
+    arguments = parser.parse_args()
+
+    exclude_globs = []
+    if arguments.exclude_globs:
+        for line in open(arguments.exclude_globs):
+            exclude_globs.append(line.strip())
+
+    linted_filenames = []
+    for path in lintutils.get_sources(arguments.source_dir, exclude_globs):
+        linted_filenames.append(str(path))
+
+    cmd = [
+        arguments.cpplint_binary,
+        '--verbose=2',
+        '--linelength=120',
+        '--filter=' + ','.join(_filters)
+    ]
+    if (arguments.cpplint_binary.endswith('.py') and
+            platform.system() == 'Windows'):
+        # Windows doesn't support executable scripts; execute with
+        # sys.executable
+        cmd.insert(0, sys.executable)
+    if arguments.quiet:
+        cmd.append('--quiet')
+    else:
+        print("\n".join(map(lambda x: "Linting {}".format(x),
+                            linted_filenames)))
+
+    # lint files in chunks: each invocation of cpplint will process 16 files
+    chunks = lintutils.chunk(linted_filenames, 16)
+    cmds = [cmd + some for some in chunks]
+    results = lintutils.run_parallel(cmds, stdout=PIPE, stderr=STDOUT)
+
+    error = False
+    # record completed processes (keyed by the first filename in the input
+    # chunk) for lookup in _check_some_files
+    completed_processes = {
+        _get_chunk_key(filenames): result
+        for filenames, result in zip(chunks, results)
+    }
+    checker = partial(_check_some_files, completed_processes)
+    pool = mp.Pool()
+    try:
+        # scan the outputs of various cpplint invocations in parallel to
+        # distill a list of problematic files
+        for problem_files, stdout in pool.imap(checker, chunks):
+            if problem_files:
+                if isinstance(stdout, bytes):
+                    stdout = stdout.decode('utf8')
+                print(stdout, file=sys.stderr)
+                error = True
+    except Exception:
+        error = True
+        raise
+    finally:
+        pool.terminate()
+        pool.join()
+
+    sys.exit(1 if error else 0)
+

cpp/build.sh

@@ -7,12 +7,19 @@ MAKE_CLEAN="OFF"
 BUILD_COVERAGE="OFF"
 DB_PATH="/opt/milvus"
 PROFILING="OFF"
-BUILD_FAISS_WITH_MKL="OFF"
 USE_JFROG_CACHE="OFF"
+RUN_CPPLINT="OFF"
+CUDA_COMPILER=/usr/local/cuda/bin/nvcc
 
-while getopts "p:d:t:uhrcgmj" arg
+while getopts "p:d:t:ulrcgjh" arg
 do
         case $arg in
+             p)
+                INSTALL_PREFIX=$OPTARG
+                ;;
+             d)
+                DB_PATH=$OPTARG
+                ;;
              t)
                 BUILD_TYPE=$OPTARG # BUILD_TYPE
                 ;;
@@ -20,11 +27,8 @@ do
                 echo "Build and run unittest cases" ;
                 BUILD_UNITTEST="ON";
                 ;;
-             p)
-                INSTALL_PREFIX=$OPTARG
-                ;;
-             d)
-                DB_PATH=$OPTARG
+             l)
+                RUN_CPPLINT="ON"
                 ;;
              r)
                 if [[ -d cmake_build ]]; then
@@ -38,9 +42,6 @@ do
              g)
                 PROFILING="ON"
                 ;;
-             m)
-                BUILD_FAISS_WITH_MKL="ON"
-                ;;
              j)
                 USE_JFROG_CACHE="ON"
                 ;;
@@ -48,65 +49,86 @@ do
                 echo "
 
 parameter:
--t: build type(default: Debug)
--u: building unit test options(default: OFF)
 -p: install prefix(default: $(pwd)/milvus)
 -d: db path(default: /opt/milvus)
+-t: build type(default: Debug)
+-u: building unit test options(default: OFF)
+-l: run cpplint, clang-format and clang-tidy(default: OFF)
 -r: remove previous build directory(default: OFF)
 -c: code coverage(default: OFF)
 -g: profiling(default: OFF)
--m: build faiss with MKL(default: OFF)
--j: use jfrog cache build directory
+-j: use jfrog cache build directory(default: OFF)
+-h: help
 
 usage:
-./build.sh -t \${BUILD_TYPE} [-u] [-h] [-g] [-r] [-c] [-k] [-m] [-j]
+./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} [-u] [-l] [-r] [-c] [-g] [-j] [-h]
                 "
                 exit 0
                 ;;
              ?)
-                echo "unknown argument"
+                echo "ERROR! unknown argument"
         exit 1
         ;;
         esac
 done
 
 if [[ ! -d cmake_build ]]; then
-	mkdir cmake_build
-	MAKE_CLEAN="ON"
+    mkdir cmake_build
 fi
 
 cd cmake_build
 
-CUDA_COMPILER=/usr/local/cuda/bin/nvcc
+CMAKE_CMD="cmake \
+-DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
+-DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}
+-DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
+-DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
+-DBUILD_COVERAGE=${BUILD_COVERAGE} \
+-DMILVUS_DB_PATH=${DB_PATH} \
+-DMILVUS_ENABLE_PROFILING=${PROFILING} \
+-DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
+../"
+echo ${CMAKE_CMD}
+${CMAKE_CMD}
 
 if [[ ${MAKE_CLEAN} == "ON" ]]; then
-    CMAKE_CMD="cmake -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
-    -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}
-    -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
-    -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
-    -DBUILD_COVERAGE=${BUILD_COVERAGE} \
-    -DMILVUS_DB_PATH=${DB_PATH} \
-    -DMILVUS_ENABLE_PROFILING=${PROFILING} \
-    -DBUILD_FAISS_WITH_MKL=${BUILD_FAISS_WITH_MKL} \
-    -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
-    -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
-    ../"
-    echo ${CMAKE_CMD}
-
-    ${CMAKE_CMD}
     make clean
 fi
 
-make -j 4 || exit 1
+if [[ ${RUN_CPPLINT} == "ON" ]]; then
+    # cpplint check
+    make lint
+    if [ $? -ne 0 ]; then
+        echo "ERROR! cpplint check not pass"
+        exit 1
+    fi
+    # clang-format check
+    make check-clang-format
+    if [ $? -ne 0 ]; then
+        echo "ERROR! clang-format check failed"
+        exit 1
+    fi
+    # clang-tidy check
+    make check-clang-tidy
+    if [ $? -ne 0 ]; then
+        echo "ERROR! clang-tidy check failed"
+        exit 1
+    fi
+else
+    # compile and build
+    make -j 4 || exit 1
 
-if [[ ${BUILD_TYPE} != "Debug" ]]; then
-    strip src/milvus_server
-fi
+    # strip binary symbol
+    if [[ ${BUILD_TYPE} != "Debug" ]]; then
+        strip src/milvus_server
+    fi
 
-make install || exit 1
+    make install || exit 1
 
-if [[ ${BUILD_COVERAGE} == "ON" ]]; then
-    cd -
-    bash `pwd`/coverage.sh
-    cd -
-fi
+    # evaluate code coverage
+    if [[ ${BUILD_COVERAGE} == "ON" ]]; then
+        cd -
+        bash `pwd`/coverage.sh
+        cd -
+    fi
+fi
\ No newline at end of file

cpp/cmake/DefineOptions.cmake

@@ -110,6 +110,7 @@ endif()
 #----------------------------------------------------------------------
 set_option_category("Test and benchmark")
 
+unset(MILVUS_BUILD_TESTS CACHE)
 if (BUILD_UNIT_TEST)
     define_option(MILVUS_BUILD_TESTS "Build the MILVUS googletest unit tests" ON)
 else()

cpp/cmake/ThirdPartyPackages.cmake

@@ -137,9 +137,26 @@ if(NOT DEFINED USE_JFROG_CACHE)
     set(USE_JFROG_CACHE "OFF")
 endif()
 if(USE_JFROG_CACHE STREQUAL "ON")
-    set(JFROG_ARTFACTORY_CACHE_URL "http://192.168.1.201:80/artifactory/generic-local/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${MILVUS_BUILD_ARCH}/${BUILD_TYPE}")
-    set(JFROG_USER_NAME "test")
-    set(JFROG_PASSWORD "Fantast1c")
+    if(DEFINED ENV{JFROG_ARTFACTORY_URL})
+        set(JFROG_ARTFACTORY_URL "$ENV{JFROG_ARTFACTORY_URL}")
+    endif()
+    if(NOT DEFINED JFROG_ARTFACTORY_URL)
+        message(FATAL_ERROR "JFROG_ARTFACTORY_URL is not set")
+    endif()
+    set(JFROG_ARTFACTORY_CACHE_URL "${JFROG_ARTFACTORY_URL}/generic-local/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${MILVUS_BUILD_ARCH}/${BUILD_TYPE}")
+    if(DEFINED ENV{JFROG_USER_NAME})
+        set(JFROG_USER_NAME "$ENV{JFROG_USER_NAME}")
+    endif()
+    if(NOT DEFINED JFROG_USER_NAME)
+        message(FATAL_ERROR "JFROG_USER_NAME is not set")
+    endif()
+    if(DEFINED ENV{JFROG_PASSWORD})
+        set(JFROG_PASSWORD "$ENV{JFROG_PASSWORD}")
+    endif()
+    if(NOT DEFINED JFROG_PASSWORD)
+        message(FATAL_ERROR "JFROG_PASSWORD is not set")
+    endif()
+
     set(THIRDPARTY_PACKAGE_CACHE "${THIRDPARTY_DIR}/cache")
 endif()
 
@@ -242,14 +259,6 @@ foreach(_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT})
     set(${_LIB_NAME} "${_LIB_VERSION}")
 endforeach()
 
-if(DEFINED ENV{MILVUS_ARROW_URL})
-    set(ARROW_SOURCE_URL "$ENV{MILVUS_ARROW_URL}")
-else()
-    set(ARROW_SOURCE_URL
-            "https://github.com/youny626/arrow.git"
-            )
-endif()
-
 if(DEFINED ENV{MILVUS_BOOST_URL})
     set(BOOST_SOURCE_URL "$ENV{MILVUS_BOOST_URL}")
 else()

cpp/conf/log_config.template

 * GLOBAL:
     FORMAT                  =   "%datetime | %level | %logger | %msg"
-    FILENAME                =   "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-global.log"
+    FILENAME                =   "@MILVUS_DB_PATH@/logs/milvus-%datetime{%y-%M-%d-%H:%m}-global.log"
     ENABLED                 =   true
     TO_FILE                 =   true
     TO_STANDARD_OUTPUT      =   false
@@ -8,12 +8,12 @@
     PERFORMANCE_TRACKING    =   false
     MAX_LOG_FILE_SIZE       =   209715200 ## Throw log files away after 200MB
 * DEBUG:
-    FILENAME                =   "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-debug.log"
+    FILENAME                =   "@MILVUS_DB_PATH@/logs/milvus-%datetime{%y-%M-%d-%H:%m}-debug.log"
     ENABLED                 =   true
 * WARNING:
-    FILENAME                =   "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-warning.log"
+    FILENAME                =   "@MILVUS_DB_PATH@/logs/milvus-%datetime{%y-%M-%d-%H:%m}-warning.log"
 * TRACE:
-    FILENAME                =   "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-trace.log"
+    FILENAME                =   "@MILVUS_DB_PATH@/logs/milvus-%datetime{%y-%M-%d-%H:%m}-trace.log"
 * VERBOSE:
     FORMAT                  =   "%datetime{%d/%M/%y} | %level-%vlevel | %msg"
     TO_FILE                 =   false
@@ -21,7 +21,7 @@
 ## Error logs
 * ERROR:
     ENABLED                 =   true
-    FILENAME                =   "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-error.log"
+    FILENAME                =   "@MILVUS_DB_PATH@/logs/milvus-%datetime{%y-%M-%d-%H:%m}-error.log"
 * FATAL:
     ENABLED                 =   true
-    FILENAME                =   "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-fatal.log"
\ No newline at end of file
+    FILENAME                =   "@MILVUS_DB_PATH@/logs/milvus-%datetime{%y-%M-%d-%H:%m}-fatal.log"
\ No newline at end of file

cpp/conf/server_config.template

 server_config:
-  address: 0.0.0.0            # milvus server ip address (IPv4)
-  port: 19530                 # the port milvus listen to, default: 19530, range: 1025 ~ 65534
-  mode: single                # milvus deployment type: single, cluster, read_only
-  time_zone: UTC+8            # Use the UTC-x or UTC+x to specify a time zone. eg. UTC+8 for China Standard Time
+  address: 0.0.0.0              # milvus server ip address (IPv4)
+  port: 19530                   # port range: 1025 ~ 65534
+  mode: single                  # deployment type: single, cluster, read_only
+  time_zone: UTC+8
 
 db_config:
-  db_path: @MILVUS_DB_PATH@             # milvus data storage path
-  db_slave_path:                        # secondry data storage path, split by semicolon
+  path: @MILVUS_DB_PATH@        # milvus database path
+  slave_path:                   # secondary database path, split by semicolon
 
   # URI format: dialect://username:password@host:port/database
   # All parts except dialect are optional, but you MUST include the delimiters
   # Currently dialect supports mysql or sqlite
-  db_backend_url: sqlite://:@:/
+  backend_url: sqlite://:@:/
 
-  archive_disk_threshold: 0        # triger archive action if storage size exceed this value, 0 means no limit, unit: GB
-  archive_days_threshold: 0        # files older than x days will be archived, 0 means no limit, unit: day
-  insert_buffer_size: 4            # maximum insert buffer size allowed, default: 4, unit: GB, should be at least 1 GB.
-                                   # the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory, unit: GB
-  build_index_gpu: 0               # which gpu is used to build index, default: 0, range: 0 ~ gpu number - 1
+  archive_disk_threshold: 0     # GB, file will be archived when disk usage exceed, 0 for no limit
+  archive_days_threshold: 0     # DAYS, older files will be archived, 0 for no limit
+  buffer_size: 4                # GB, maximum insert buffer size allowed
+  build_index_gpu: 0            # gpu id used for building index
 
 metric_config:
-  is_startup: off                       # if monitoring start: on, off
-  collector: prometheus                 # metrics collector: prometheus
-  prometheus_config:                    # following are prometheus configure
-    port: 8080                          # the port prometheus use to fetch metrics
-    push_gateway_ip_address: 127.0.0.1  # push method configure: push gateway ip address
-    push_gateway_port: 9091             # push method configure: push gateway port
+  auto_bootup: off              # whether enable monitoring when bootup
+  collector: prometheus         # prometheus
+  prometheus_config:
+    port: 8080                  # port prometheus used to fetch metrics
 
 cache_config:
-  cpu_cache_capacity: 16            # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory
-  cpu_cache_free_percent: 0.85      # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0
-  insert_cache_immediately: false   # insert data will be load into cache immediately for hot query
+  cpu_mem_capacity: 16          # GB, CPU memory size used for cache
+  cpu_mem_threshold: 0.85       # percent of data kept when cache cleanup triggered
+  cache_insert_data: false      # whether load data into cache when insert
 
 engine_config:
-  use_blas_threshold: 20
+  blas_threshold: 20
 
 resource_config:
   mode: simple
-  resources:
-#    - cpu
+  pool:
+    - cpu
     - gpu0

cpp/src/cache/CpuCacheMgr.cpp

@@ -17,7 +17,7 @@
 
 
 #include "CpuCacheMgr.h"
-#include "server/ServerConfig.h"
+#include "server/Config.h"
 #include "utils/Log.h"
 
 namespace zilliz {
@@ -29,17 +29,27 @@ namespace {
 }
 
 CpuCacheMgr::CpuCacheMgr() {
-    server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE);
-    int64_t cap = config.GetInt64Value(server::CONFIG_CPU_CACHE_CAPACITY, 16);
-    cap *= unit;
+    server::Config& config = server::Config::GetInstance();
+    Status s;
+
+    int32_t cpu_mem_cap;
+    s = config.GetCacheConfigCpuMemCapacity(cpu_mem_cap);
+    if (!s.ok()) {
+        SERVER_LOG_ERROR << s.message();
+    }
+    int64_t cap = cpu_mem_cap * unit;
     cache_ = std::make_shared<Cache<DataObjPtr>>(cap, 1UL<<32);
 
-    double free_percent = config.GetDoubleValue(server::CACHE_FREE_PERCENT, 0.85);
-    if(free_percent > 0.0 && free_percent <= 1.0) {
-        cache_->set_freemem_percent(free_percent);
+    float cpu_mem_threshold;
+    s = config.GetCacheConfigCpuMemThreshold(cpu_mem_threshold);
+    if (!s.ok()) {
+        SERVER_LOG_ERROR << s.message();
+    }
+    if (cpu_mem_threshold > 0.0 && cpu_mem_threshold <= 1.0) {
+        cache_->set_freemem_percent(cpu_mem_threshold);
     } else {
-        SERVER_LOG_ERROR << "Invalid cache_free_percent: " << free_percent <<
-         ", defaultly set to " << cache_->freemem_percent();
+        SERVER_LOG_ERROR << "Invalid cpu_mem_threshold: " << cpu_mem_threshold
+                         << ", by default set to " << cache_->freemem_percent();
     }
 }
 

cpp/src/cache/DataObj.h

@@ -18,7 +18,7 @@
 
 #pragma once
 
-#include "src/wrapper/vec_index.h"
+#include "src/wrapper/VecIndex.h"
 
 #include <memory>
 

cpp/src/cache/GpuCacheMgr.cpp

@@ -19,7 +19,7 @@
 #include <sstream>
 #include "utils/Log.h"
 #include "GpuCacheMgr.h"
-#include "server/ServerConfig.h"
+#include "server/Config.h"
 
 namespace zilliz {
 namespace milvus {
@@ -33,18 +33,27 @@ namespace {
 }
 
 GpuCacheMgr::GpuCacheMgr() {
-    server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE);
+    server::Config& config = server::Config::GetInstance();
+    Status s;
 
-    int64_t cap = config.GetInt64Value(server::CONFIG_GPU_CACHE_CAPACITY, 0);
-    cap *= G_BYTE;
+    int32_t gpu_mem_cap;
+    s = config.GetCacheConfigGpuMemCapacity(gpu_mem_cap);
+    if (!s.ok()) {
+        SERVER_LOG_ERROR << s.message();
+    }
+    int32_t cap = gpu_mem_cap * G_BYTE;
     cache_ = std::make_shared<Cache<DataObjPtr>>(cap, 1UL<<32);
 
-    double free_percent = config.GetDoubleValue(server::GPU_CACHE_FREE_PERCENT, 0.85);
-    if (free_percent > 0.0 && free_percent <= 1.0) {
-        cache_->set_freemem_percent(free_percent);
+    float gpu_mem_threshold;
+    s = config.GetCacheConfigGpuMemThreshold(gpu_mem_threshold);
+    if (!s.ok()) {
+        SERVER_LOG_ERROR << s.message();
+    }
+    if (gpu_mem_threshold > 0.0 && gpu_mem_threshold <= 1.0) {
+        cache_->set_freemem_percent(gpu_mem_threshold);
     } else {
-        SERVER_LOG_ERROR << "Invalid gpu_cache_free_percent: " << free_percent <<
-                         ", defaultly set to " << cache_->freemem_percent();
+        SERVER_LOG_ERROR << "Invalid gpu_mem_threshold: " << gpu_mem_threshold
+                         << ", by default set to " << cache_->freemem_percent();
     }
 }
 

cpp/src/cache/LRU.h

@@ -34,78 +34,78 @@ public:
     typedef typename std::list<key_value_pair_t>::iterator list_iterator_t;
     typedef typename std::list<key_value_pair_t>::reverse_iterator reverse_list_iterator_t;
 
-    LRU(size_t max_size) : _max_size(max_size) {}
+    LRU(size_t max_size) : max_size_(max_size) {}
 
     void put(const key_t& key, const value_t& value) {
-        auto it = _cache_items_map.find(key);
-        _cache_items_list.push_front(key_value_pair_t(key, value));
-        if (it != _cache_items_map.end()) {
-            _cache_items_list.erase(it->second);
-            _cache_items_map.erase(it);
+        auto it = cache_items_map_.find(key);
+        cache_items_list_.push_front(key_value_pair_t(key, value));
+        if (it != cache_items_map_.end()) {
+            cache_items_list_.erase(it->second);
+            cache_items_map_.erase(it);
         }
-        _cache_items_map[key] = _cache_items_list.begin();
+        cache_items_map_[key] = cache_items_list_.begin();
 
-        if (_cache_items_map.size() > _max_size) {
-            auto last = _cache_items_list.end();
+        if (cache_items_map_.size() > max_size_) {
+            auto last = cache_items_list_.end();
             last--;
-            _cache_items_map.erase(last->first);
-            _cache_items_list.pop_back();
+            cache_items_map_.erase(last->first);
+            cache_items_list_.pop_back();
         }
     }
 
     const value_t& get(const key_t& key) {
-        auto it = _cache_items_map.find(key);
-        if (it == _cache_items_map.end()) {
+        auto it = cache_items_map_.find(key);
+        if (it == cache_items_map_.end()) {
             throw std::range_error("There is no such key in cache");
         } else {
-            _cache_items_list.splice(_cache_items_list.begin(), _cache_items_list, it->second);
+            cache_items_list_.splice(cache_items_list_.begin(), cache_items_list_, it->second);
             return it->second->second;
         }
     }
 
     void erase(const key_t& key) {
-        auto it = _cache_items_map.find(key);
-        if (it != _cache_items_map.end()) {
-            _cache_items_list.erase(it->second);
-            _cache_items_map.erase(it);
+        auto it = cache_items_map_.find(key);
+        if (it != cache_items_map_.end()) {
+            cache_items_list_.erase(it->second);
+            cache_items_map_.erase(it);
         }
     }
 
     bool exists(const key_t& key) const {
-        return _cache_items_map.find(key) != _cache_items_map.end();
+        return cache_items_map_.find(key) != cache_items_map_.end();
     }
 
     size_t size() const {
-        return _cache_items_map.size();
+        return cache_items_map_.size();
     }
 
     list_iterator_t begin() {
-        _iter = _cache_items_list.begin();
-        return _iter;
+        iter_ = cache_items_list_.begin();
+        return iter_;
     }
 
     list_iterator_t end() {
-        return _cache_items_list.end();
+        return cache_items_list_.end();
     }
 
     reverse_list_iterator_t rbegin() {
-        return _cache_items_list.rbegin();
+        return cache_items_list_.rbegin();
     }
 
     reverse_list_iterator_t rend() {
-        return _cache_items_list.rend();
+        return cache_items_list_.rend();
     }
 
     void clear() {
-        _cache_items_list.clear();
-        _cache_items_map.clear();
+        cache_items_list_.clear();
+        cache_items_map_.clear();
     }
 
 private:
-    std::list<key_value_pair_t> _cache_items_list;
-    std::unordered_map<key_t, list_iterator_t> _cache_items_map;
-    size_t _max_size;
-    list_iterator_t _iter;
+    std::list<key_value_pair_t> cache_items_list_;
+    std::unordered_map<key_t, list_iterator_t> cache_items_map_;
+    size_t max_size_;
+    list_iterator_t iter_;
 };
 
 }   // cache

cpp/src/core/cmake/ThirdPartyPackagesCore.cmake

@@ -123,9 +123,7 @@ if(NOT DEFINED USE_JFROG_CACHE)
     set(USE_JFROG_CACHE "OFF")
 endif()
 if(USE_JFROG_CACHE STREQUAL "ON")
-    set(JFROG_ARTFACTORY_CACHE_URL "http://192.168.1.201:80/artifactory/generic-local/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${KNOWHERE_BUILD_ARCH}/${BUILD_TYPE}")
-    set(JFROG_USER_NAME "test")
-    set(JFROG_PASSWORD "Fantast1c")
+    set(JFROG_ARTFACTORY_CACHE_URL "${JFROG_ARTFACTORY_URL}/generic-local/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${KNOWHERE_BUILD_ARCH}/${BUILD_TYPE}")
     set(THIRDPARTY_PACKAGE_CACHE "${THIRDPARTY_DIR}/cache")
 endif()
 

cpp/src/db/DBFactory.cpp

@@ -36,7 +36,7 @@ namespace engine {
 DBOptions DBFactory::BuildOption() {
     auto meta = MetaFactory::BuildOption();
         DBOptions options;
-    options.meta = meta;
+    options.meta_ = meta;
     return options;
 }
 

cpp/src/db/DBImpl.cpp

@@ -56,7 +56,7 @@ DBImpl::DBImpl(const DBOptions& options)
       shutting_down_(true),
       compact_thread_pool_(1, 1),
       index_thread_pool_(1, 1) {
-    meta_ptr_ = MetaFactory::Build(options.meta, options.mode);
+    meta_ptr_ = MetaFactory::Build(options.meta_, options.mode_);
     mem_mgr_ = MemManagerFactory::Build(meta_ptr_, options_);
     Start();
 }
@@ -77,7 +77,7 @@ Status DBImpl::Start() {
     shutting_down_.store(false, std::memory_order_release);
 
     //for distribute version, some nodes are read only
-    if (options_.mode != DBOptions::MODE::READ_ONLY) {
+    if (options_.mode_ != DBOptions::MODE::READ_ONLY) {
         ENGINE_LOG_TRACE << "StartTimerTasks";
         bg_timer_thread_ = std::thread(&DBImpl::BackgroundTimerTask, this);
     }
@@ -98,7 +98,7 @@ Status DBImpl::Stop() {
     //wait compaction/buildindex finish
     bg_timer_thread_.join();
 
-    if (options_.mode != DBOptions::MODE::READ_ONLY) {
+    if (options_.mode_ != DBOptions::MODE::READ_ONLY) {
         meta_ptr_->CleanUp();
     }
 
@@ -133,9 +133,9 @@ Status DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& date
         meta_ptr_->DeleteTable(table_id); //soft delete table
 
         //scheduler will determine when to delete table files
-        auto nres = ResMgrInst::GetInstance()->GetNumOfComputeResource();
+        auto nres = scheduler::ResMgrInst::GetInstance()->GetNumOfComputeResource();
         scheduler::DeleteJobPtr job = std::make_shared<scheduler::DeleteJob>(0, table_id, meta_ptr_, nres);
-        JobMgrInst::GetInstance()->Put(job);
+        scheduler::JobMgrInst::GetInstance()->Put(job);
         job->WaitAndDelete();
     } else {
         meta_ptr_->DropPartitionsByDates(table_id, dates);
@@ -649,7 +649,7 @@ Status DBImpl::BackgroundMergeFiles(const std::string& table_id) {
     bool has_merge = false;
     for (auto& kv : raw_files) {
         auto files = kv.second;
-        if (files.size() < options_.merge_trigger_number) {
+        if (files.size() < options_.merge_trigger_number_) {
             ENGINE_LOG_DEBUG << "Files number not greater equal than merge trigger number, skip merge action";
             continue;
         }
@@ -684,7 +684,7 @@ void DBImpl::BackgroundCompaction(std::set<std::string> table_ids) {
     meta_ptr_->Archive();
 
     int ttl = 5*meta::M_SEC;//default: file will be deleted after 5 minutes
-    if (options_.mode == DBOptions::MODE::CLUSTER) {
+    if (options_.mode_ == DBOptions::MODE::CLUSTER) {
         ttl = meta::D_SEC;
     }
     meta_ptr_->CleanUpFilesWithTTL(ttl);

cpp/src/db/Options.h

@@ -52,10 +52,10 @@ private:
 };
 
 struct DBMetaOptions {
-    std::string path;
-    std::vector<std::string> slave_paths;
-    std::string backend_uri;
-    ArchiveConf archive_conf = ArchiveConf("delete");
+    std::string path_;
+    std::vector<std::string> slave_paths_;
+    std::string backend_uri_;
+    ArchiveConf archive_conf_ = ArchiveConf("delete");
 }; // DBMetaOptions
 
 struct DBOptions {
@@ -65,11 +65,11 @@ struct DBOptions {
         READ_ONLY
     } MODE;
 
-    uint16_t  merge_trigger_number = 2;
-    DBMetaOptions meta;
-    int mode = MODE::SINGLE;
+    uint16_t  merge_trigger_number_ = 2;
+    DBMetaOptions meta_;
+    int mode_ = MODE::SINGLE;
 
-    size_t insert_buffer_size = 4 * ONE_GB;
+    size_t insert_buffer_size_ = 4 * ONE_GB;
     bool insert_cache_immediately_ = false;
 }; // Options
 

cpp/src/db/Utils.cpp

@@ -21,6 +21,7 @@
 
 #include <mutex>
 #include <chrono>
+#include <regex>
 #include <boost/filesystem.hpp>
 
 namespace zilliz {
@@ -42,8 +43,8 @@ std::string ConstructParentFolder(const std::string& db_path, const meta::TableF
 }
 
 std::string GetTableFileParentFolder(const DBMetaOptions& options, const meta::TableFileSchema& table_file) {
-    uint64_t path_count = options.slave_paths.size() + 1;
-    std::string target_path = options.path;
+    uint64_t path_count = options.slave_paths_.size() + 1;
+    std::string target_path = options.path_;
     uint64_t index = 0;
 
     if(meta::TableFileSchema::NEW_INDEX == table_file.file_type_) {
@@ -60,7 +61,7 @@ std::string GetTableFileParentFolder(const DBMetaOptions& options, const meta::T
     }
 
     if (index > 0) {
-        target_path = options.slave_paths[index - 1];
+        target_path = options.slave_paths_[index - 1];
     }
 
     return ConstructParentFolder(target_path, table_file);
@@ -77,7 +78,7 @@ long GetMicroSecTimeStamp() {
 }
 
 Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id) {
-    std::string db_path = options.path;
+    std::string db_path = options.path_;
     std::string table_path = db_path + TABLES_FOLDER + table_id;
     auto status = server::CommonUtil::CreateDirectory(table_path);
     if (!status.ok()) {
@@ -85,7 +86,7 @@ Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id
         return status;
     }
 
-    for(auto& path : options.slave_paths) {
+    for(auto& path : options.slave_paths_) {
         table_path = path + TABLES_FOLDER + table_id;
         status = server::CommonUtil::CreateDirectory(table_path);
         if (!status.ok()) {
@@ -98,8 +99,8 @@ Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id
 }
 
 Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool force) {
-    std::vector<std::string> paths = options.slave_paths;
-    paths.push_back(options.path);
+    std::vector<std::string> paths = options.slave_paths_;
+    paths.push_back(options.path_);
 
     for(auto& path : paths) {
         std::string table_path = path + TABLES_FOLDER + table_id;
@@ -131,13 +132,13 @@ Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema&
 }
 
 Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) {
-    std::string parent_path = ConstructParentFolder(options.path, table_file);
+    std::string parent_path = ConstructParentFolder(options.path_, table_file);
     std::string file_path = parent_path + "/" + table_file.file_id_;
     if(boost::filesystem::exists(file_path)) {
         table_file.location_ = file_path;
         return Status::OK();
     } else {
-        for(auto& path : options.slave_paths) {
+        for(auto& path : options.slave_paths_) {
             parent_path = ConstructParentFolder(path, table_file);
             file_path = parent_path + "/" + table_file.file_id_;
             if(boost::filesystem::exists(file_path)) {
@@ -148,7 +149,7 @@ Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& tab
     }
 
     std::string msg = "Table file doesn't exist: " + file_path;
-    ENGINE_LOG_ERROR << msg << " in path: " << options.path
+    ENGINE_LOG_ERROR << msg << " in path: " << options.path_
                      << " for table: " << table_file.table_id_;
 
     return Status(DB_ERROR, msg);
@@ -195,6 +196,41 @@ meta::DateT GetDate() {
     return GetDate(std::time(nullptr), 0);
 }
 
+// URI format: dialect://username:password@host:port/database
+Status ParseMetaUri(const std::string& uri, MetaUriInfo& info) {
+    std::string dialect_regex = "(.*)";
+    std::string username_tegex = "(.*)";
+    std::string password_regex = "(.*)";
+    std::string host_regex = "(.*)";
+    std::string port_regex = "(.*)";
+    std::string db_name_regex = "(.*)";
+    std::string uri_regex_str =
+            dialect_regex + "\\:\\/\\/" +
+            username_tegex + "\\:" +
+            password_regex + "\\@" +
+            host_regex + "\\:" +
+            port_regex + "\\/" +
+            db_name_regex;
+
+    std::regex uri_regex(uri_regex_str);
+    std::smatch pieces_match;
+
+    if (std::regex_match(uri, pieces_match, uri_regex)) {
+        info.dialect_ = pieces_match[1].str();
+        info.username_ = pieces_match[2].str();
+        info.password_ = pieces_match[3].str();
+        info.host_ = pieces_match[4].str();
+        info.port_ = pieces_match[5].str();
+        info.db_name_ = pieces_match[6].str();
+
+        //TODO: verify host, port...
+    } else {
+        return Status(DB_INVALID_META_URI, "Invalid meta uri: " + uri);
+    }
+
+    return Status::OK();
+}
+
 } // namespace utils
 } // namespace engine
 } // namespace milvus

cpp/src/db/Utils.h

@@ -44,6 +44,17 @@ meta::DateT GetDate(const std::time_t &t, int day_delta = 0);
 meta::DateT GetDate();
 meta::DateT GetDateWithDelta(int day_delta);
 
+struct MetaUriInfo {
+    std::string dialect_;
+    std::string username_;
+    std::string password_;
+    std::string host_;
+    std::string port_;
+    std::string db_name_;
+};
+
+Status ParseMetaUri(const std::string& uri, MetaUriInfo& info);
+
 } // namespace utils
 } // namespace engine
 } // namespace milvus

cpp/src/db/engine/ExecutionEngineImpl.cpp

@@ -23,9 +23,10 @@
 #include "utils/CommonUtil.h"
 #include "utils/Exception.h"
 
-#include "src/wrapper/vec_index.h"
-#include "src/wrapper/vec_impl.h"
+#include "src/wrapper/VecIndex.h"
+#include "src/wrapper/VecImpl.h"
 #include "knowhere/common/Exception.h"
+#include "server/Config.h"
 
 #include <stdexcept>
 
@@ -326,9 +327,9 @@ Status ExecutionEngineImpl::GpuCache(uint64_t gpu_id) {
 // TODO(linxj): remove.
 Status ExecutionEngineImpl::Init() {
     using namespace zilliz::milvus::server;
-    ServerConfig &config = ServerConfig::GetInstance();
-    ConfigNode server_config = config.GetConfig(CONFIG_DB);
-    gpu_num_ = server_config.GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, 0);
+    server::Config &config = server::Config::GetInstance();
+    Status s = config.GetDBConfigBuildIndexGPU(gpu_num_);
+    if (!s.ok()) return s;
 
     return Status::OK();
 }

cpp/src/db/engine/ExecutionEngineImpl.h

@@ -18,7 +18,7 @@
 #pragma once
 
 #include "ExecutionEngine.h"
-#include "src/wrapper/vec_index.h"
+#include "src/wrapper/VecIndex.h"
 
 #include <memory>
 #include <string>
@@ -101,7 +101,7 @@ protected:
     std::string location_;
 
     int32_t nlist_ = 0;
-    int64_t gpu_num_ = 0;
+    int32_t gpu_num_ = 0;
 };
 
 

cpp/src/db/insert/MemManagerImpl.cpp

@@ -43,7 +43,7 @@ Status MemManagerImpl::InsertVectors(const std::string &table_id_,
                                     const float *vectors_,
                                     IDNumbers &vector_ids_) {
 
-    while (GetCurrentMem() > options_.insert_buffer_size) {
+    while (GetCurrentMem() > options_.insert_buffer_size_) {
         std::this_thread::sleep_for(std::chrono::milliseconds(1));
     }
 

cpp/src/db/meta/MetaFactory.cpp

@@ -20,13 +20,14 @@
 #include "MySQLMetaImpl.h"
 #include "utils/Log.h"
 #include "utils/Exception.h"
+#include "db/Utils.h"
 
 #include <stdlib.h>
 #include <time.h>
 #include <sstream>
 #include <cstdlib>
 #include <string>
-#include <regex>
+#include <string.h>
 
 namespace zilliz {
 namespace milvus {
@@ -42,45 +43,30 @@ namespace engine {
         }
 
         DBMetaOptions meta;
-        meta.path = p;
+        meta.path_ = p;
         return meta;
     }
 
     meta::MetaPtr MetaFactory::Build(const DBMetaOptions &metaOptions, const int &mode) {
-        std::string uri = metaOptions.backend_uri;
+        std::string uri = metaOptions.backend_uri_;
 
-        std::string dialectRegex = "(.*)";
-        std::string usernameRegex = "(.*)";
-        std::string passwordRegex = "(.*)";
-        std::string hostRegex = "(.*)";
-        std::string portRegex = "(.*)";
-        std::string dbNameRegex = "(.*)";
-        std::string uriRegexStr = dialectRegex + "\\:\\/\\/" +
-                                  usernameRegex + "\\:" +
-                                  passwordRegex + "\\@" +
-                                  hostRegex + "\\:" +
-                                  portRegex + "\\/" +
-                                  dbNameRegex;
-        std::regex uriRegex(uriRegexStr);
-        std::smatch pieces_match;
-
-        if (std::regex_match(uri, pieces_match, uriRegex)) {
-            std::string dialect = pieces_match[1].str();
-            std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower);
-            if (dialect.find("mysql") != std::string::npos) {
-                ENGINE_LOG_INFO << "Using MySQL";
-                return std::make_shared<meta::MySQLMetaImpl>(metaOptions, mode);
-            } else if (dialect.find("sqlite") != std::string::npos) {
-                ENGINE_LOG_INFO << "Using SQLite";
-                return std::make_shared<meta::SqliteMetaImpl>(metaOptions);
-            } else {
-                ENGINE_LOG_ERROR << "Invalid dialect in URI: dialect = " << dialect;
-                throw InvalidArgumentException("URI dialect is not mysql / sqlite");
-            }
-        } else {
+        utils::MetaUriInfo uri_info;
+        auto status = utils::ParseMetaUri(uri, uri_info);
+        if(!status.ok()) {
             ENGINE_LOG_ERROR << "Wrong URI format: URI = " << uri;
             throw InvalidArgumentException("Wrong URI format ");
         }
+
+        if (strcasecmp(uri_info.dialect_.c_str(), "mysql") == 0) {
+            ENGINE_LOG_INFO << "Using MySQL";
+            return std::make_shared<meta::MySQLMetaImpl>(metaOptions, mode);
+        } else if (strcasecmp(uri_info.dialect_.c_str(), "sqlite") == 0) {
+            ENGINE_LOG_INFO << "Using SQLite";
+            return std::make_shared<meta::SqliteMetaImpl>(metaOptions);
+        } else {
+            ENGINE_LOG_ERROR << "Invalid dialect in URI: dialect = " << uri_info.dialect_;
+            throw InvalidArgumentException("URI dialect is not mysql / sqlite");
+        }
     }
 
 } // namespace engine

cpp/src/db/meta/MySQLMetaImpl.h

@@ -103,13 +103,16 @@ class MySQLMetaImpl : public Meta {
     Status NextFileId(std::string &file_id);
     Status NextTableId(std::string &table_id);
     Status DiscardFiles(long long to_discard_size);
+
+    void ValidateMetaSchema();
     Status Initialize();
 
+private:
     const DBMetaOptions options_;
     const int mode_;
 
     std::shared_ptr<MySQLConnectionPool> mysql_connection_pool_;
-    bool safe_grab = false;
+    bool safe_grab_ = false;
 
 //        std::mutex connectionMutex_;
 }; // DBMetaImpl

cpp/src/db/meta/SqliteMetaImpl.cpp

@@ -84,7 +84,6 @@ inline auto StoragePrototype(const std::string &path) {
 
 using ConnectorT = decltype(StoragePrototype(""));
 static std::unique_ptr<ConnectorT> ConnectorPtr;
-using ConditionT = decltype(c(&TableFileSchema::id_) == 1UL);
 
 SqliteMetaImpl::SqliteMetaImpl(const DBMetaOptions &options_)
     : options_(options_) {
@@ -111,29 +110,37 @@ Status SqliteMetaImpl::NextFileId(std::string &file_id) {
     return Status::OK();
 }
 
-Status SqliteMetaImpl::Initialize() {
-    if (!boost::filesystem::is_directory(options_.path)) {
-        auto ret = boost::filesystem::create_directory(options_.path);
-        if (!ret) {
-            std::string msg = "Failed to create db directory " + options_.path;
-            ENGINE_LOG_ERROR << msg;
-            return Status(DB_INVALID_PATH, msg);
-        }
+void SqliteMetaImpl::ValidateMetaSchema() {
+    if(ConnectorPtr == nullptr) {
+        return;
     }
 
-    ConnectorPtr = std::make_unique<ConnectorT>(StoragePrototype(options_.path + "/meta.sqlite"));
-
     //old meta could be recreated since schema changed, throw exception if meta schema is not compatible
     auto ret = ConnectorPtr->sync_schema_simulate();
     if(ret.find(META_TABLES) != ret.end()
-        && sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLES]) {
-            throw Exception(DB_INCOMPATIB_META, "Meta schema is created by Milvus old version");
+       && sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLES]) {
+        throw Exception(DB_INCOMPATIB_META, "Meta Tables schema is created by Milvus old version");
     }
     if(ret.find(META_TABLEFILES) != ret.end()
        && sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLEFILES]) {
-        throw Exception(DB_INCOMPATIB_META, "Meta schema is created by Milvus old version");
+        throw Exception(DB_INCOMPATIB_META, "Meta TableFiles schema is created by Milvus old version");
+    }
+}
+
+Status SqliteMetaImpl::Initialize() {
+    if (!boost::filesystem::is_directory(options_.path_)) {
+        auto ret = boost::filesystem::create_directory(options_.path_);
+        if (!ret) {
+            std::string msg = "Failed to create db directory " + options_.path_;
+            ENGINE_LOG_ERROR << msg;
+            return Status(DB_INVALID_PATH, msg);
+        }
     }
 
+    ConnectorPtr = std::make_unique<ConnectorT>(StoragePrototype(options_.path_ + "/meta.sqlite"));
+
+    ValidateMetaSchema();
+
     ConnectorPtr->sync_schema();
     ConnectorPtr->open_forever(); // thread safe option
     ConnectorPtr->pragma.journal_mode(journal_mode::WAL); // WAL => write ahead log
@@ -879,7 +886,7 @@ Status SqliteMetaImpl::GetTableFiles(const std::string& table_id,
 
 // PXU TODO: Support Swap
 Status SqliteMetaImpl::Archive() {
-    auto &criterias = options_.archive_conf.GetCriterias();
+    auto &criterias = options_.archive_conf_.GetCriterias();
     if (criterias.size() == 0) {
         return Status::OK();
     }

cpp/src/db/meta/SqliteMetaImpl.h

@@ -97,10 +97,12 @@ class SqliteMetaImpl : public Meta {
     Status NextFileId(std::string &file_id);
     Status NextTableId(std::string &table_id);
     Status DiscardFiles(long to_discard_size);
+
+    void ValidateMetaSchema();
     Status Initialize();
 
+private:
     const DBMetaOptions options_;
-
     std::mutex meta_mutex_;
 }; // DBMetaImpl
 

cpp/src/grpc/gen-milvus/milvus.grpc.pb.cc

@@ -53,7 +53,7 @@ MilvusService::Stub::Stub(const std::shared_ptr< ::grpc::ChannelInterface>& chan
   , rpcmethod_SearchInFiles_(MilvusService_method_names[6], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
   , rpcmethod_DescribeTable_(MilvusService_method_names[7], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
   , rpcmethod_CountTable_(MilvusService_method_names[8], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
-  , rpcmethod_ShowTables_(MilvusService_method_names[9], ::grpc::internal::RpcMethod::SERVER_STREAMING, channel)
+  , rpcmethod_ShowTables_(MilvusService_method_names[9], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
   , rpcmethod_Cmd_(MilvusService_method_names[10], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
   , rpcmethod_DeleteByRange_(MilvusService_method_names[11], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
   , rpcmethod_PreloadTable_(MilvusService_method_names[12], ::grpc::internal::RpcMethod::NORMAL_RPC, channel)
@@ -313,20 +313,32 @@ void MilvusService::Stub::experimental_async::CountTable(::grpc::ClientContext*
   return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TableRowCount>::Create(channel_.get(), cq, rpcmethod_CountTable_, context, request, false);
 }
 
-::grpc::ClientReader< ::milvus::grpc::TableName>* MilvusService::Stub::ShowTablesRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request) {
-  return ::grpc_impl::internal::ClientReaderFactory< ::milvus::grpc::TableName>::Create(channel_.get(), rpcmethod_ShowTables_, context, request);
+::grpc::Status MilvusService::Stub::ShowTables(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::milvus::grpc::TableNameList* response) {
+  return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_ShowTables_, context, request, response);
 }
 
-void MilvusService::Stub::experimental_async::ShowTables(::grpc::ClientContext* context, ::milvus::grpc::Command* request, ::grpc::experimental::ClientReadReactor< ::milvus::grpc::TableName>* reactor) {
-  ::grpc_impl::internal::ClientCallbackReaderFactory< ::milvus::grpc::TableName>::Create(stub_->channel_.get(), stub_->rpcmethod_ShowTables_, context, request, reactor);
+void MilvusService::Stub::experimental_async::ShowTables(::grpc::ClientContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::TableNameList* response, std::function<void(::grpc::Status)> f) {
+  ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_ShowTables_, context, request, response, std::move(f));
 }
 
-::grpc::ClientAsyncReader< ::milvus::grpc::TableName>* MilvusService::Stub::AsyncShowTablesRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq, void* tag) {
-  return ::grpc_impl::internal::ClientAsyncReaderFactory< ::milvus::grpc::TableName>::Create(channel_.get(), cq, rpcmethod_ShowTables_, context, request, true, tag);
+void MilvusService::Stub::experimental_async::ShowTables(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TableNameList* response, std::function<void(::grpc::Status)> f) {
+  ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_ShowTables_, context, request, response, std::move(f));
 }
 
-::grpc::ClientAsyncReader< ::milvus::grpc::TableName>* MilvusService::Stub::PrepareAsyncShowTablesRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) {
-  return ::grpc_impl::internal::ClientAsyncReaderFactory< ::milvus::grpc::TableName>::Create(channel_.get(), cq, rpcmethod_ShowTables_, context, request, false, nullptr);
+void MilvusService::Stub::experimental_async::ShowTables(::grpc::ClientContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::TableNameList* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
+  ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_ShowTables_, context, request, response, reactor);
+}
+
+void MilvusService::Stub::experimental_async::ShowTables(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TableNameList* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
+  ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_ShowTables_, context, request, response, reactor);
+}
+
+::grpc::ClientAsyncResponseReader< ::milvus::grpc::TableNameList>* MilvusService::Stub::AsyncShowTablesRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) {
+  return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TableNameList>::Create(channel_.get(), cq, rpcmethod_ShowTables_, context, request, true);
+}
+
+::grpc::ClientAsyncResponseReader< ::milvus::grpc::TableNameList>* MilvusService::Stub::PrepareAsyncShowTablesRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) {
+  return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TableNameList>::Create(channel_.get(), cq, rpcmethod_ShowTables_, context, request, false);
 }
 
 ::grpc::Status MilvusService::Stub::Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::milvus::grpc::StringReply* response) {
@@ -517,8 +529,8 @@ MilvusService::Service::Service() {
           std::mem_fn(&MilvusService::Service::CountTable), this)));
   AddMethod(new ::grpc::internal::RpcServiceMethod(
       MilvusService_method_names[9],
-      ::grpc::internal::RpcMethod::SERVER_STREAMING,
-      new ::grpc::internal::ServerStreamingHandler< MilvusService::Service, ::milvus::grpc::Command, ::milvus::grpc::TableName>(
+      ::grpc::internal::RpcMethod::NORMAL_RPC,
+      new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::Command, ::milvus::grpc::TableNameList>(
           std::mem_fn(&MilvusService::Service::ShowTables), this)));
   AddMethod(new ::grpc::internal::RpcServiceMethod(
       MilvusService_method_names[10],
@@ -613,10 +625,10 @@ MilvusService::Service::~Service() {
   return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
 }
 
-::grpc::Status MilvusService::Service::ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::grpc::ServerWriter< ::milvus::grpc::TableName>* writer) {
+::grpc::Status MilvusService::Service::ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::TableNameList* response) {
   (void) context;
   (void) request;
-  (void) writer;
+  (void) response;
   return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
 }
 

cpp/src/grpc/milvus.proto

@@ -8,18 +8,26 @@ package milvus.grpc;
  * @brief Table Name
  */
 message TableName {
+    string table_name = 1;
+}
+
+/**
+ * @brief Table Name List
+ */
+message TableNameList {
     Status status = 1;
-    string table_name = 2;
+    repeated string table_names = 2;
 }
 
 /**
  * @brief Table Schema
  */
 message TableSchema {
-    TableName table_name = 1;
-    int64 dimension = 2;
-    int64 index_file_size = 3;
-    int32 metric_type = 4;
+    Status status = 1;
+    string table_name = 2;
+    int64 dimension = 3;
+    int64 index_file_size = 4;
+    int32 metric_type = 5;
 }
 
 /**
@@ -141,8 +149,9 @@ message Index {
  * @brief Index params
  */
 message IndexParam {
-    TableName table_name = 1;
-    Index index = 2;
+    Status status = 1;
+    string table_name = 2;
+    Index index = 3;
 }
 
 /**
@@ -264,7 +273,7 @@ service MilvusService {
      *
      * @return table names.
      */
-    rpc ShowTables(Command) returns (stream TableName) {}
+    rpc ShowTables(Command) returns (TableNameList) {}
 
     /**
      * @brief Give the server status

cpp/src/main.cpp

@@ -110,7 +110,7 @@ main(int argc, char *argv[]) {
     signal(SIGUSR2, server::SignalUtil::HandleSignal);
     signal(SIGTERM, server::SignalUtil::HandleSignal);
 
-    server::Server &server = server::Server::Instance();
+    server::Server &server = server::Server::GetInstance();
     server.Init(start_daemonized, pid_filename, config_filename, log_config_file);
     server.Start();
 

cpp/src/metrics/MetricBase.h

@@ -19,7 +19,6 @@
 #pragma once
 
 #include "utils/Error.h"
-#include "server/ServerConfig.h"
 #include "SystemInfo.h"
 
 namespace zilliz {
@@ -83,11 +82,6 @@ class MetricsBase{
     virtual void CPUTemperature() {};
 };
 
-
-
-
-
-
 }
 }
 }
\ No newline at end of file

cpp/src/scheduler/Algorithm.cpp

@@ -20,7 +20,7 @@
 
 namespace zilliz {
 namespace milvus {
-namespace engine {
+namespace scheduler {
 
 constexpr uint64_t MAXINT = std::numeric_limits<uint32_t >::max();