update README files

.gitignore

@@ -26,3 +26,6 @@ cmake_build
 *.lo
 *.tar.gz
 *.log
+.coverage
+*.pyc
+cov_html/

.travis.yml

+language: cpp
+sudo: required
+dist: bionic
+
+cache:
+  directories:
+    - $HOME/.ccache
+
+addons:
+  apt:
+    update: true
+
+before_install:
+  - source ci/travis/before-install.sh
+
+install:
+  - source $TRAVIS_BUILD_DIR/ci/travis/install_dependency.sh
+
+script:
+  - $TRAVIS_BUILD_DIR/ci/travis/travis_build.sh

CHANGELOG.md

 # Changelog
 
 Please mark all change in change log and use the ticket from JIRA.
+# Milvus 0.6.0 (TODO)
+
+## Bug
+- \#228 - memory usage increased slowly during searching vectors
+- \#246 - Exclude src/external folder from code coverage for jenkin ci
+- \#248 - Reside src/external in thirdparty
+- \#316 - Some files not merged after vectors added
+- \#327 - Search does not use GPU when index type is FLAT
+- \#340 - Test cases run failed on 0.6.0
+- \#353 - Rename config.h.in to version.h.in
+- \#374 - sdk_simple return empty result
+- \#377 - Create partition success if tag name only contains spaces
+- \#397 - sdk_simple return incorrect result
+- \#399 - Create partition should be failed if partition tag existed
+- \#412 - Message returned is confused when partition created with null partition name
+- \#416 - Drop the same partition success repeatally
+- \#440 - Query API in customization still uses old version
+
+## Feature
+- \#12 - Pure CPU version for Milvus
+- \#77 - Support table partition
+- \#127 - Support new Index type IVFPQ
+- \#226 - Experimental shards middleware for Milvus
+- \#227 - Support new index types SPTAG-KDT and SPTAG-BKT
+- \#346 - Support build index with multiple gpu
+
+## Improvement
+- \#255 - Add ivfsq8 test report detailed version
+- \#260 - C++ SDK README
+- \#266 - Rpc request source code refactor
+- \#275 - Rename C++ SDK IndexType
+- \#284 - Change C++ SDK to shared library
+- \#306 - Use int64 for all config integer
+- \#310 - Add Q&A for 'protocol https not supported or disable in libcurl' issue
+- \#314 - add Find FAISS in CMake
+- \#322 - Add option to enable / disable prometheus
+- \#358 - Add more information in build.sh and install.md
+- \#404 - Add virtual method Init() in Pass abstract class
+- \#409 - Add a Fallback pass in optimizer
+- \#433 - C++ SDK query result is not easy to use
+- \#449 - Add ShowPartitions example for C++ SDK
+
+## Task
+
 # Milvus 0.5.3 (2019-11-13)
 
 ## Bug
@@ -87,7 +131,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-658 - Fix SQ8 Hybrid can't search
 - MS-665 - IVF_SQ8H search crash when no GPU resource in search_resources
 - \#9 - Change default gpu_cache_capacity to 4
-- \#20 - C++ sdk example get grpc error 
+- \#20 - C++ sdk example get grpc error
 - \#23 - Add unittest to improve code coverage
 - \#31 - make clang-format failed after run build.sh -l
 - \#39 - Create SQ8H index hang if using github server version
@@ -139,7 +183,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-635 - Add compile option to support customized faiss
 - MS-660 - add ubuntu_build_deps.sh
 - \#18 - Add all test cases
-	
+
 # Milvus 0.4.0 (2019-09-12)
 
 ## Bug
@@ -348,11 +392,11 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-82 - Update server startup welcome message
 - MS-83 - Update vecwise to Milvus
 - MS-77 - Performance issue of post-search action
-- MS-22 - Enhancement for MemVector size control 
+- MS-22 - Enhancement for MemVector size control
 - MS-92 - Unify behavior of debug and release build
 - MS-98 - Install all unit test to installation directory
 - MS-115 - Change is_startup of metric_config switch from true to on
-- MS-122 - Archive criteria config 
+- MS-122 - Archive criteria config
 - MS-124 - HasTable interface
 - MS-126 - Add more error code
 - MS-128 - Change default db path

README.md

@@ -78,6 +78,4 @@ Below is a list of Milvus contributors. We greatly appreciate your contributions
 
 ## License
 
-[Apache License 2.0](LICENSE)
-
-
+[Apache License 2.0](LICENSE)
\ No newline at end of file

README_CN.md

@@ -78,4 +78,3 @@ Milvus 提供稳定的 [Python](https://github.com/milvus-io/pymilvus)、[Java](
 ## 许可协议
 
 [Apache 许可协议2.0版](https://github.com/milvus-io/milvus/blob/master/LICENSE)
-

README_JP.md

@@ -72,4 +72,4 @@ C++サンプルコードを実行するために、次のコマンドをつか
 
 ## ライセンス
 
-[Apache 2.0ライセンス](LICENSE)
+[Apache 2.0ライセンス](LICENSE)
\ No newline at end of file

ci/jenkins/Jenkinsfile

@@ -2,7 +2,7 @@
 
 String cron_timezone = "TZ=Asia/Shanghai"
 String cron_string = BRANCH_NAME == "master" ? "H 0 * * * " : ""
-cron_string =  BRANCH_NAME == "0.5.2" ? "H 1 * * * " : cron_string
+cron_string =  BRANCH_NAME == "0.6.0" ? "H 1 * * * " : cron_string
 
 pipeline {
     agent none
@@ -33,71 +33,136 @@ pipeline {
     }
 
     stages {
-        stage("Ubuntu 18.04") {
+        stage("Ubuntu 18.04 x86_64") {
             environment {
                 OS_NAME = "ubuntu18.04"
-                PACKAGE_VERSION = VersionNumber([
-                    versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-ubuntu18.04-x86_64-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}'
-                ]);
-                DOCKER_VERSION = "${SEMVER}-${OS_NAME}-${LOWER_BUILD_TYPE}"
+                CPU_ARCH = "amd64"
             }
 
-            stages {
-                stage("Run Build") {
-                    agent {
-                        kubernetes {
-                            label 'build'
-                            defaultContainer 'jnlp'
-                            yamlFile 'ci/jenkins/pod/milvus-build-env-pod.yaml'
-                        }
+            parallel {
+                stage ("GPU Version") {
+                    environment {
+                        BINRARY_VERSION = "gpu"
+                        PACKAGE_VERSION = VersionNumber([
+                            versionNumberString : '${SEMVER}-gpu-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}'
+                        ]);
+                        DOCKER_VERSION = "${SEMVER}-gpu-${OS_NAME}-${LOWER_BUILD_TYPE}"
                     }
 
                     stages {
-                        stage('Build') {
-                            steps {
-                                container('milvus-build-env') {
-                                    script {
-                                        load "${env.WORKSPACE}/ci/jenkins/step/build.groovy"
-                                    }
+                        stage("Run Build") {
+                            agent {
+                                kubernetes {
+                                    label "${BINRARY_VERSION}-build"
+                                    defaultContainer 'jnlp'
+                                    yamlFile 'ci/jenkins/pod/milvus-gpu-version-build-env-pod.yaml'
                                 }
                             }
-                        }
-                        stage('Code Coverage') {
-                            steps {
-                                container('milvus-build-env') {
-                                    script {
-                                        load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy"
+
+                            stages {
+                                stage('Build') {
+                                    steps {
+                                        container('milvus-build-env') {
+                                            script {
+                                                load "${env.WORKSPACE}/ci/jenkins/step/build.groovy"
+                                            }
+                                        }
+                                    }
+                                }
+                                stage('Code Coverage') {
+                                    steps {
+                                        container('milvus-build-env') {
+                                            script {
+                                                load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy"
+                                            }
+                                        }
+                                    }
+                                }
+                                stage('Upload Package') {
+                                    steps {
+                                        container('milvus-build-env') {
+                                            script {
+                                                load "${env.WORKSPACE}/ci/jenkins/step/package.groovy"
+                                            }
+                                        }
                                     }
                                 }
                             }
                         }
-                        stage('Upload Package') {
-                            steps {
-                                container('milvus-build-env') {
-                                    script {
-                                        load "${env.WORKSPACE}/ci/jenkins/step/packaging.groovy"
+
+                        stage("Publish docker images") {
+                            agent {
+                                kubernetes {
+                                    label "${BINRARY_VERSION}-publish"
+                                    defaultContainer 'jnlp'
+                                    yamlFile 'ci/jenkins/pod/docker-pod.yaml'
+                                }
+                            }
+
+                            stages {
+                                stage('Publish') {
+                                    steps {
+                                        container('publish-images'){
+                                            script {
+                                                load "${env.WORKSPACE}/ci/jenkins/step/publishImages.groovy"
+                                            }
+                                        }
                                     }
                                 }
                             }
                         }
-                    }
-                }
 
-                stage("Publish docker images") {
-                    agent {
-                        kubernetes {
-                            label 'publish'
-                            defaultContainer 'jnlp'
-                            yamlFile 'ci/jenkins/pod/docker-pod.yaml'
-                        }
-                    }
+                        stage("Deploy to Development") {
+                            agent {
+                                kubernetes {
+                                    label "${BINRARY_VERSION}-dev-test"
+                                    defaultContainer 'jnlp'
+                                    yamlFile 'ci/jenkins/pod/testEnvironment.yaml'
+                                }
+                            }
 
-                    stages {
-                        stage('Publish') {
-                            steps {
-                                container('publish-images'){
-                                    script {
-                                        load "${env.WORKSPACE}/ci/jenkins/step/publishImages.groovy"
+                            stages {
+                                stage("Deploy to Dev") {
+                                    steps {
+                                        container('milvus-test-env') {
+                                            script {
+                                                load "${env.WORKSPACE}/ci/jenkins/step/deploySingle2Dev.groovy"
+                                            }
+                                        }
+                                    }
+                                }
+
+                                stage("Dev Test") {
+                                    steps {
+                                        container('milvus-test-env') {
+                                            script {
+                                                boolean isNightlyTest = isTimeTriggeredBuild()
+                                                if (isNightlyTest) {
+                                                    load "${env.WORKSPACE}/ci/jenkins/step/singleDevNightlyTest.groovy"
+                                                } else {
+                                                    load "${env.WORKSPACE}/ci/jenkins/step/singleDevTest.groovy"
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
+
+                                stage ("Cleanup Dev") {
+                                    steps {
+                                        container('milvus-test-env') {
+                                            script {
+                                                load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy"
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                            post {
+                                unsuccessful {
+                                    container('milvus-test-env') {
+                                        script {
+                                            load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy"
+                                        }
                                     }
                                 }
                             }
@@ -105,56 +170,130 @@ pipeline {
                     }
                 }
 
-                stage("Deploy to Development") {
-                    agent {
-                        kubernetes {
-                            label 'dev-test'
-                            defaultContainer 'jnlp'
-                            yamlFile 'ci/jenkins/pod/testEnvironment.yaml'
-                        }
+                stage ("CPU Version") {
+                    environment {
+                        BINRARY_VERSION = "cpu"
+                        PACKAGE_VERSION = VersionNumber([
+                            versionNumberString : '${SEMVER}-cpu-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}'
+                        ]);
+                        DOCKER_VERSION = "${SEMVER}-cpu-${OS_NAME}-${LOWER_BUILD_TYPE}"
                     }
 
                     stages {
-                        stage("Deploy to Dev") {
-                            steps {
-                                container('milvus-test-env') {
-                                    script {
-                                        load "${env.WORKSPACE}/ci/jenkins/step/deploySingle2Dev.groovy"
+                        stage("Run Build") {
+                            agent {
+                                kubernetes {
+                                    label "${BINRARY_VERSION}-build"
+                                    defaultContainer 'jnlp'
+                                    yamlFile 'ci/jenkins/pod/milvus-cpu-version-build-env-pod.yaml'
+                                }
+                            }
+
+                            stages {
+                                stage('Build') {
+                                    steps {
+                                        container('milvus-build-env') {
+                                            script {
+                                                load "${env.WORKSPACE}/ci/jenkins/step/build.groovy"
+                                            }
+                                        }
+                                    }
+                                }
+                                stage('Code Coverage') {
+                                    steps {
+                                        container('milvus-build-env') {
+                                            script {
+                                                load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy"
+                                            }
+                                        }
+                                    }
+                                }
+                                stage('Upload Package') {
+                                    steps {
+                                        container('milvus-build-env') {
+                                            script {
+                                                load "${env.WORKSPACE}/ci/jenkins/step/package.groovy"
+                                            }
+                                        }
                                     }
                                 }
                             }
                         }
 
-                        stage("Dev Test") {
-                            steps {
-                                container('milvus-test-env') {
-                                    script {
-                                        boolean isNightlyTest = isTimeTriggeredBuild()
-                                        if (isNightlyTest) {
-                                            load "${env.WORKSPACE}/ci/jenkins/step/singleDevNightlyTest.groovy"
-                                        } else {
-                                            load "${env.WORKSPACE}/ci/jenkins/step/singleDevTest.groovy"
+                        stage("Publish docker images") {
+                            agent {
+                                kubernetes {
+                                    label "${BINRARY_VERSION}-publish"
+                                    defaultContainer 'jnlp'
+                                    yamlFile 'ci/jenkins/pod/docker-pod.yaml'
+                                }
+                            }
+
+                            stages {
+                                stage('Publish') {
+                                    steps {
+                                        container('publish-images'){
+                                            script {
+                                                load "${env.WORKSPACE}/ci/jenkins/step/publishImages.groovy"
+                                            }
                                         }
                                     }
                                 }
                             }
                         }
 
-                        stage ("Cleanup Dev") {
-                            steps {
-                                container('milvus-test-env') {
-                                    script {
-                                        load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy"
+                        stage("Deploy to Development") {
+                            agent {
+                                kubernetes {
+                                    label "${BINRARY_VERSION}-dev-test"
+                                    defaultContainer 'jnlp'
+                                    yamlFile 'ci/jenkins/pod/testEnvironment.yaml'
+                                }
+                            }
+
+                            stages {
+                                stage("Deploy to Dev") {
+                                    steps {
+                                        container('milvus-test-env') {
+                                            script {
+                                                load "${env.WORKSPACE}/ci/jenkins/step/deploySingle2Dev.groovy"
+                                            }
+                                        }
+                                    }
+                                }
+
+                                stage("Dev Test") {
+                                    steps {
+                                        container('milvus-test-env') {
+                                            script {
+                                                boolean isNightlyTest = isTimeTriggeredBuild()
+                                                if (isNightlyTest) {
+                                                    load "${env.WORKSPACE}/ci/jenkins/step/singleDevNightlyTest.groovy"
+                                                } else {
+                                                    load "${env.WORKSPACE}/ci/jenkins/step/singleDevTest.groovy"
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
+
+                                stage ("Cleanup Dev") {
+                                    steps {
+                                        container('milvus-test-env') {
+                                            script {
+                                                load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy"
+                                            }
+                                        }
                                     }
                                 }
                             }
-                        }
-                    }
-                    post {
-                        unsuccessful {
-                            container('milvus-test-env') {
-                                script {
-                                    load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy"
+                            post {
+                                unsuccessful {
+                                    container('milvus-test-env') {
+                                        script {
+                                            load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy"
+                                        }
+                                    }
                                 }
                             }
                         }

ci/jenkins/pod/milvus-cpu-version-build-env-pod.yaml

+apiVersion: v1
+kind: Pod
+metadata:
+  name: milvus-cpu-build-env
+  labels:
+    app: milvus
+    componet: cpu-build-env
+spec:
+  containers:
+  - name: milvus-build-env
+    image: registry.zilliz.com/milvus/milvus-cpu-build-env:v0.6.0-ubuntu18.04
+    env:
+    - name: POD_IP
+      valueFrom:
+        fieldRef:
+          fieldPath: status.podIP
+    command:
+    - cat
+    tty: true
+    resources:
+      limits:
+        memory: "32Gi"
+        cpu: "8.0"
+      requests:
+        memory: "16Gi"
+        cpu: "4.0"
+  - name: milvus-mysql
+    image: mysql:5.6
+    env:
+    - name: MYSQL_ROOT_PASSWORD
+      value: 123456
+    ports:
+    - containerPort: 3306
+      name: mysql

ci/jenkins/pod/milvus-build-env-pod.yaml → ci/jenkins/pod/milvus-gpu-version-build-env-pod.yaml

 apiVersion: v1
 kind: Pod
 metadata:
-  name: milvus-build-env
+  name: milvus-gpu-build-env
   labels:
     app: milvus
-    componet: build-env
+    componet: gpu-build-env
 spec:
   containers:
   - name: milvus-build-env
-    image: registry.zilliz.com/milvus/milvus-build-env:v0.5.1-ubuntu18.04
+    image: registry.zilliz.com/milvus/milvus-gpu-build-env:v0.6.0-ubuntu18.04
     env:
     - name: POD_IP
       valueFrom:

ci/jenkins/scripts/packaging.sh

-#!/usr/bin/env bash
-
-set -ex
-
-pip3 install -r requirements.txt
-./yaml_processor.py merge -f /opt/milvus/conf/server_config.yaml -m ../yaml/update_server_config.yaml -i && \
-rm /opt/milvus/conf/server_config.yaml.bak
-
-if [ -d "/opt/milvus/unittest" ]; then
-  rm -rf "/opt/milvus/unittest"
-fi
-
-tar -zcvf ./${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz -C /opt/ milvus
\ No newline at end of file

ci/jenkins/step/build.groovy

 timeout(time: 60, unit: 'MINUTES') {
-    dir ("ci/jenkins/scripts") {
-        sh "./build.sh -l"
+    dir ("ci/scripts") {
         withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
-            sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -d /opt/milvus -j -u -c"
+            if ("${env.BINRARY_VERSION}" == "gpu") {
+            	sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -g -j -u -c"
+        	} else {
+        		sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -m -j -u -c"
+        	}
         }
     }
 }
-

ci/jenkins/step/cleanupSingleDev.groovy

 try {
-    def helmResult = sh script: "helm status ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu", returnStatus: true
+    def helmResult = sh script: "helm status ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}", returnStatus: true
     if (!helmResult) {
-        sh "helm del --purge ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu"
+        sh "helm del --purge ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}"
     }
 } catch (exc) {
-    def helmResult = sh script: "helm status ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu", returnStatus: true
+    def helmResult = sh script: "helm status ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}", returnStatus: true
     if (!helmResult) {
-        sh "helm del --purge ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu"
+        sh "helm del --purge ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}"
     }
     throw exc
 }

ci/jenkins/step/coverage.groovy

 timeout(time: 30, unit: 'MINUTES') {
-    dir ("ci/jenkins/scripts") {
+    dir ("ci/scripts") {
         sh "./coverage.sh -o /opt/milvus -u root -p 123456 -t \$POD_IP"
         // Set some env variables so codecov detection script works correctly
         withCredentials([[$class: 'StringBinding', credentialsId: "${env.PIPELINE_NAME}-codecov-token", variable: 'CODECOV_TOKEN']]) {

ci/jenkins/step/deploySingle2Dev.groovy

 sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
 sh 'helm repo update'
 dir ('milvus-helm') {
-    checkout([$class: 'GitSCM', branches: [[name: "0.5.3"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.3:refs/remotes/origin/0.5.3"]]])
-    dir ("milvus-gpu") {
-        sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/db_backend/sqlite_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
+    checkout([$class: 'GitSCM', branches: [[name: "0.6.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.6.0:refs/remotes/origin/0.6.0"]]])
+    dir ("milvus") {
+        sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/sqlite_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
     }
 }
 

ci/jenkins/step/package.groovy

+timeout(time: 5, unit: 'MINUTES') {
+    dir ("ci/jenkins/scripts") {
+        sh "pip3 install -r requirements.txt"
+        sh "./yaml_processor.py merge -f /opt/milvus/conf/server_config.yaml -m ../yaml/update_server_config.yaml -i && rm /opt/milvus/conf/server_config.yaml.bak"
+    }
+    sh "tar -zcvf ./${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz -C /opt/ milvus"
+    withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) {
+        def uploadStatus = sh(returnStatus: true, script: "curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -T ./${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz ${params.JFROG_ARTFACTORY_URL}/milvus/package/${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz")
+        if (uploadStatus != 0) {
+            error("\" ${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz \" upload to \" ${params.JFROG_ARTFACTORY_URL}/milvus/package/${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz \" failed!")
+        }
+    }
+}

ci/jenkins/step/packaging.groovy

-timeout(time: 5, unit: 'MINUTES') {
-    dir ("ci/jenkins/scripts") {
-        sh "./packaging.sh"
-        withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) {
-            def uploadStatus = sh(returnStatus: true, script: "curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -T ./${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz ${params.JFROG_ARTFACTORY_URL}/milvus/package/${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz")
-            if (uploadStatus != 0) {
-                error("\" ${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz \" upload to \" ${params.JFROG_ARTFACTORY_URL}/milvus/package/${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz \" failed!")
-            }
-        }
-    }
-}

ci/jenkins/step/publishImages.groovy

 container('publish-images') {
     timeout(time: 15, unit: 'MINUTES') {
-        dir ("docker/deploy/${OS_NAME}") {
+        dir ("docker/deploy/${env.BINRARY_VERSION}/${env.OS_NAME}") {
             def binaryPackage = "${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz"
 
             withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) {

ci/jenkins/step/singleDevNightlyTest.groovy

 timeout(time: 90, unit: 'MINUTES') {
     dir ("tests/milvus_python_test") {
         sh 'python3 -m pip install -r requirements.txt'
-        sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu-milvus-gpu-engine.milvus.svc.cluster.local"
+        sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-engine.milvus.svc.cluster.local"
     }
     // mysql database backend test
-    load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy"
+    load "ci/jenkins/jenkinsfile/cleanupSingleDev.groovy"
 
     if (!fileExists('milvus-helm')) {
         dir ("milvus-helm") {
-            checkout([$class: 'GitSCM', branches: [[name: "0.5.3"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.3:refs/remotes/origin/0.5.3"]]])
+            checkout([$class: 'GitSCM', branches: [[name: "0.6.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.6.0:refs/remotes/origin/0.6.0"]]])
         }
     }
     dir ("milvus-helm") {
-        dir ("milvus-gpu") {
-            sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
+        dir ("milvus") {
+            sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
         }
     }
     dir ("tests/milvus_python_test") {
-        sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu-milvus-gpu-engine.milvus.svc.cluster.local"
+        sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-engine.milvus.svc.cluster.local"
     }
 }

ci/jenkins/step/singleDevTest.groovy

 timeout(time: 60, unit: 'MINUTES') {
     dir ("tests/milvus_python_test") {
         sh 'python3 -m pip install -r requirements.txt'
-        sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu-milvus-gpu-engine.milvus.svc.cluster.local"
+        sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-engine.milvus.svc.cluster.local"
     }
-    // mysql database backend test
-    // load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy"
 
-    // Remove mysql-version tests: 10-28
+    // mysql database backend test
+    // load "ci/jenkins/jenkinsfile/cleanupSingleDev.groovy"
 
     // if (!fileExists('milvus-helm')) {
     //     dir ("milvus-helm") {
-    //         checkout([$class: 'GitSCM', branches: [[name: "0.5.3"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.3:refs/remotes/origin/0.5.3"]]])
+    //         checkout([$class: 'GitSCM', branches: [[name: "0.6.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.6.0:refs/remotes/origin/0.6.0"]]])
     //     }
     // }
     // dir ("milvus-helm") {
-    //     dir ("milvus-gpu") {
-    //         sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
+    //     dir ("milvus") {
+    //         sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
     //     }
     // }
     // dir ("tests/milvus_python_test") {
-    //     sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu-milvus-gpu-engine.milvus.svc.cluster.local"
+    //     sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-engine.milvus.svc.cluster.local"
     // }
 }

ci/jenkinsfile/packaged_milvus.groovy

@@ -5,9 +5,6 @@ container('milvus-build-env') {
                 gitlabCommitStatus(name: 'Packaged Engine') {
                     if (fileExists('milvus')) {
                         try {
-                            if (fileExists('milvus/unittest')) {
-                                sh "rm -rf ./milvus/unittest"
-                            }
                             sh "tar -zcvf ./${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz ./milvus"
                             def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy"
                             fileTransfer.FileTransfer("${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz", "${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}", 'nas storage')

ci/jenkins/scripts/build.sh → ci/scripts/build.sh

 #!/bin/bash
 
+set -e
+
 SOURCE="${BASH_SOURCE[0]}"
 while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
   DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
@@ -8,38 +10,41 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
 done
 SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
 
-CMAKE_BUILD_DIR="${SCRIPTS_DIR}/../../../core/cmake_build"
+MILVUS_CORE_DIR="${SCRIPTS_DIR}/../../core"
+CORE_BUILD_DIR="${MILVUS_CORE_DIR}/cmake_build"
 BUILD_TYPE="Debug"
 BUILD_UNITTEST="OFF"
 INSTALL_PREFIX="/opt/milvus"
+FAISS_ROOT=""
+CUSTOMIZATION="OFF" # default use origin faiss
 BUILD_COVERAGE="OFF"
-DB_PATH="/opt/milvus"
-PROFILING="OFF"
 USE_JFROG_CACHE="OFF"
 RUN_CPPLINT="OFF"
-CUSTOMIZATION="OFF" # default use ori faiss
+GPU_VERSION="OFF"
+WITH_MKL="OFF"
 CUDA_COMPILER=/usr/local/cuda/bin/nvcc
 
-CUSTOMIZED_FAISS_URL="${FAISS_URL:-NONE}"
-wget -q --method HEAD ${CUSTOMIZED_FAISS_URL}
-if [ $? -eq 0 ]; then
-  CUSTOMIZATION="ON"
-else
-  CUSTOMIZATION="OFF"
-fi
-
-while getopts "o:d:t:ulcgjhx" arg
+while getopts "o:t:b:f:gxulcjmh" arg
 do
         case $arg in
              o)
                 INSTALL_PREFIX=$OPTARG
                 ;;
-             d)
-                DB_PATH=$OPTARG
-                ;;
              t)
                 BUILD_TYPE=$OPTARG # BUILD_TYPE
                 ;;
+             b)
+                CORE_BUILD_DIR=$OPTARG # CORE_BUILD_DIR
+                ;;
+             f)
+                FAISS_ROOT=$OPTARG # FAISS ROOT PATH
+                ;;
+             g)
+                GPU_VERSION="ON";
+                ;;
+             x)
+                CUSTOMIZATION="ON";
+                ;;
              u)
                 echo "Build and run unittest cases" ;
                 BUILD_UNITTEST="ON";
@@ -50,31 +55,31 @@ do
              c)
                 BUILD_COVERAGE="ON"
                 ;;
-             g)
-                PROFILING="ON"
-                ;;
              j)
                 USE_JFROG_CACHE="ON"
                 ;;
-             x)
-                CUSTOMIZATION="OFF" # force use ori faiss
+             m)
+                WITH_MKL="ON"
                 ;;
              h) # help
                 echo "
 
 parameter:
 -o: install prefix(default: /opt/milvus)
--d: db data path(default: /opt/milvus)
 -t: build type(default: Debug)
+-b: core code build directory
+-f: faiss root path
+-g: gpu version
+-x: milvus customization (default: OFF)
 -u: building unit test options(default: OFF)
 -l: run cpplint, clang-format and clang-tidy(default: OFF)
 -c: code coverage(default: OFF)
--g: profiling(default: OFF)
 -j: use jfrog cache build directory(default: OFF)
+-m: build with MKL(default: OFF)
 -h: help
 
 usage:
-./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} [-u] [-l] [-r] [-c] [-g] [-j] [-h]
+./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-g] [-x] [-u] [-l] [-c] [-j] [-m] [-h]
                 "
                 exit 0
                 ;;
@@ -85,31 +90,30 @@ usage:
         esac
 done
 
-if [[ ! -d ${CMAKE_BUILD_DIR} ]]; then
-    mkdir ${CMAKE_BUILD_DIR}
+if [[ ! -d ${CORE_BUILD_DIR} ]]; then
+    mkdir ${CORE_BUILD_DIR}
 fi
 
-cd ${CMAKE_BUILD_DIR}
-
-# remove make cache since build.sh -l use default variables
-# force update the variables each time
-make rebuild_cache
+cd ${CORE_BUILD_DIR}
 
 CMAKE_CMD="cmake \
--DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
 -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}
 -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
 -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
+-DMILVUS_GPU_VERSION=${GPU_VERSION} \
+-DCUSTOMIZATION=${CUSTOMIZATION} \
+-DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
 -DBUILD_COVERAGE=${BUILD_COVERAGE} \
--DMILVUS_DB_PATH=${DB_PATH} \
--DMILVUS_ENABLE_PROFILING=${PROFILING} \
 -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
--DCUSTOMIZATION=${CUSTOMIZATION} \
--DFAISS_URL=${CUSTOMIZED_FAISS_URL} \
-.."
+-DFAISS_ROOT=${FAISS_ROOT} \
+-DFAISS_WITH_MKL=${WITH_MKL} \
+-DArrow_SOURCE=AUTO \
+-DFAISS_SOURCE=AUTO \
+${MILVUS_CORE_DIR}"
 echo ${CMAKE_CMD}
 ${CMAKE_CMD}
 
+
 if [[ ${RUN_CPPLINT} == "ON" ]]; then
     # cpplint check
     make lint
@@ -135,8 +139,8 @@ if [[ ${RUN_CPPLINT} == "ON" ]]; then
 #        exit 1
 #    fi
 #    echo "clang-tidy check passed!"
-else
-    # compile and build
-    make -j8 || exit 1
-    make install || exit 1
 fi
+
+# compile and build
+make -j8 || exit 1
+make install || exit 1

ci/jenkins/scripts/coverage.sh → ci/scripts/coverage.sh

@@ -9,18 +9,22 @@ done
 SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
 
 INSTALL_PREFIX="/opt/milvus"
-CMAKE_BUILD_DIR="${SCRIPTS_DIR}/../../../core/cmake_build"
+MILVUS_CORE_DIR="${SCRIPTS_DIR}/../../core"
+CORE_BUILD_DIR="${MILVUS_CORE_DIR}/cmake_build"
 MYSQL_USER_NAME=root
 MYSQL_PASSWORD=123456
 MYSQL_HOST='127.0.0.1'
 MYSQL_PORT='3306'
 
-while getopts "o:u:p:t:h" arg
+while getopts "o:b:u:p:t:h" arg
 do
         case $arg in
              o)
                 INSTALL_PREFIX=$OPTARG
                 ;;
+             b)
+                CORE_BUILD_DIR=$OPTARG # CORE_BUILD_DIR
+                ;;
              u)
                 MYSQL_USER_NAME=$OPTARG
                 ;;
@@ -35,13 +39,14 @@ do
 
 parameter:
 -o: milvus install prefix(default: /opt/milvus)
+-b: core code build directory
 -u: mysql account
 -p: mysql password
 -t: mysql host
 -h: help
 
 usage:
-./coverage.sh -o \${INSTALL_PREFIX} -u \${MYSQL_USER} -p \${MYSQL_PASSWORD} -t \${MYSQL_HOST} [-h]
+./coverage.sh -o \${INSTALL_PREFIX} -b \${CORE_BUILD_DIR} -u \${MYSQL_USER} -p \${MYSQL_PASSWORD} -t \${MYSQL_HOST} [-h]
                 "
                 exit 0
                 ;;
@@ -63,12 +68,14 @@ FILE_INFO_OUTPUT="output.info"
 FILE_INFO_OUTPUT_NEW="output_new.info"
 DIR_LCOV_OUTPUT="lcov_out"
 
-DIR_GCNO="${CMAKE_BUILD_DIR}"
+DIR_GCNO="${CORE_BUILD_DIR}"
 DIR_UNITTEST="${INSTALL_PREFIX}/unittest"
 
+cd ${SCRIPTS_DIR}
+
 # delete old code coverage info files
-rm -rf lcov_out
-rm -f FILE_INFO_BASE FILE_INFO_MILVUS FILE_INFO_OUTPUT FILE_INFO_OUTPUT_NEW
+rm -rf ${DIR_LCOV_OUTPUT}
+rm -f ${FILE_INFO_BASE} ${FILE_INFO_MILVUS} ${FILE_INFO_OUTPUT} ${FILE_INFO_OUTPUT_NEW}
 
 MYSQL_DB_NAME=milvus_`date +%s%N`
 
@@ -109,7 +116,7 @@ for test in `ls ${DIR_UNITTEST}`; do
     if [ $? -ne 0 ]; then
         echo ${args}
         echo ${DIR_UNITTEST}/${test} "run failed"
-        exit -1
+        exit 1
     fi
 done
 
@@ -132,12 +139,11 @@ ${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \
     "*/src/server/Server.cpp" \
     "*/src/server/DBWrapper.cpp" \
     "*/src/server/grpc_impl/GrpcServer.cpp" \
-    "*/src/external/easyloggingpp/easylogging++.h" \
-    "*/src/external/easyloggingpp/easylogging++.cc"
+    "*/thirdparty/*"
 
 if [ $? -ne 0 ]; then
     echo "gen ${FILE_INFO_OUTPUT_NEW} failed"
-    exit -2
+    exit 2
 fi
 
 # gen html report

ci/travis/before-install.sh

+#!/bin/bash
+
+set -ex
+
+if [[ "${TRAVIS_OS_NAME}" == "linux" ]]; then
+  export CCACHE_COMPRESS=1
+  export CCACHE_COMPRESSLEVEL=5
+  export CCACHE_COMPILERCHECK=content
+  export PATH=/usr/lib/ccache/:$PATH
+  ccache --show-stats
+fi
+
+set +ex

ci/travis/install_dependency.sh

+#!/usr/bin/env bash
+
+set -e
+
+wget -P /tmp https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
+
+sudo apt-key add /tmp/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
+
+echo "deb https://apt.repos.intel.com/mkl all main" | \
+  sudo tee /etc/apt/sources.list.d/intel-mkl.list
+
+sudo wget -O /usr/share/keyrings/apache-arrow-keyring.gpg https://dl.bintray.com/apache/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-keyring.gpg
+
+sudo tee /etc/apt/sources.list.d/apache-arrow.list <<APT_LINE
+deb [arch=amd64 signed-by=/usr/share/keyrings/apache-arrow-keyring.gpg] https://dl.bintray.com/apache/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/ $(lsb_release --codename --short) main
+deb-src [signed-by=/usr/share/keyrings/apache-arrow-keyring.gpg] https://dl.bintray.com/apache/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/ $(lsb_release --codename --short) main
+APT_LINE
+
+sudo apt-get update -qq
+
+sudo apt-get install -y -q --no-install-recommends \
+  gfortran \
+  lsb-core \
+  libtool \
+  automake \
+  ccache \
+  pkg-config \
+  libarrow-dev \
+  libjemalloc-dev \
+  libboost-serialization-dev \
+  libboost-filesystem-dev \
+  libboost-system-dev \
+  libboost-regex-dev \
+  intel-mkl-gnu-2019.5-281 \
+  intel-mkl-core-2019.5-281 \
+  libmysqlclient-dev \
+  clang-format-6.0 \
+  clang-tidy-6.0 \
+  lcov
+
+sudo ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so \
+  /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so
+
+export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/intel/compilers_and_libraries_2019.5.281/linux/mkl/lib/intel64

ci/travis/travis_build.sh

+#!/usr/bin/env bash
+
+set -ex
+
+source $TRAVIS_BUILD_DIR/ci/travis/travis_env_common.sh
+
+only_library_mode=no
+
+while true; do
+    case "$1" in
+	--only-library)
+	    only_library_mode=yes
+	    shift ;;
+	*) break ;;
+    esac
+done
+
+BUILD_COMMON_FLAGS="-t ${MILVUS_BUILD_TYPE} -o ${MILVUS_INSTALL_PREFIX} -b ${MILVUS_BUILD_DIR}"
+
+if [ $only_library_mode == "yes" ]; then
+  ${TRAVIS_BUILD_DIR}/ci/scripts/build.sh ${BUILD_COMMON_FLAGS} -m
+else
+  ${TRAVIS_BUILD_DIR}/ci/scripts/build.sh ${BUILD_COMMON_FLAGS} -m -u -c
+fi
\ No newline at end of file

ci/travis/travis_env_common.sh

+export MILVUS_CORE_DIR=${TRAVIS_BUILD_DIR}/core
+export MILVUS_BUILD_DIR=${TRAVIS_BUILD_DIR}/core/cmake_build
+export MILVUS_INSTALL_PREFIX=/opt/milvus
+export MILVUS_TRAVIS_COVERAGE=${MILVUS_TRAVIS_COVERAGE:=0}
+
+if [ "${MILVUS_TRAVIS_COVERAGE}" == "1" ]; then
+  export MILVUS_CPP_COVERAGE_FILE=${TRAVIS_BUILD_DIR}/output_new.info
+fi
+
+export MILVUS_BUILD_TYPE=${MILVUS_BUILD_TYPE:=Release}

core/.gitignore

 milvus/
 conf/server_config.yaml
 conf/log_config.conf
-version.h
+src/config.h
+src/version.h
 lcov_out/
 base.info
 output.info
 output_new.info
 server.info
 *.pyc
+src/grpc/python_gen.h
+src/grpc/python/

core/CMakeLists.txt

@@ -18,47 +18,42 @@
 #-------------------------------------------------------------------------------
 
 
-cmake_minimum_required(VERSION 3.14)
+cmake_minimum_required(VERSION 3.12)
 message(STATUS "Building using CMake version: ${CMAKE_VERSION}")
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 
-MACRO (GET_CURRENT_TIME CURRENT_TIME)
+MACRO(GET_CURRENT_TIME CURRENT_TIME)
     execute_process(COMMAND "date" +"%Y-%m-%d %H:%M.%S" OUTPUT_VARIABLE ${CURRENT_TIME})
-ENDMACRO (GET_CURRENT_TIME)
+ENDMACRO(GET_CURRENT_TIME)
 
 GET_CURRENT_TIME(BUILD_TIME)
 string(REGEX REPLACE "\n" "" BUILD_TIME ${BUILD_TIME})
 message(STATUS "Build time = ${BUILD_TIME}")
 
-MACRO (GET_GIT_BRANCH_NAME GIT_BRANCH_NAME)
-    execute_process(COMMAND "git" rev-parse --abbrev-ref HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
-    if(GIT_BRANCH_NAME STREQUAL "")
-        execute_process(COMMAND "git" symbolic-ref --short -q HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
-    endif()
-ENDMACRO (GET_GIT_BRANCH_NAME)
+MACRO(GET_GIT_BRANCH_NAME GIT_BRANCH_NAME)
+    execute_process(COMMAND sh "-c" "git log --decorate | head -n 1 | sed 's/.*(\\(.*\\))/\\1/' | sed 's/.* \\(.*\\),.*/\\1/' | sed 's=[a-zA-Z]*\/==g'"
+            OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
+ENDMACRO(GET_GIT_BRANCH_NAME)
 
 GET_GIT_BRANCH_NAME(GIT_BRANCH_NAME)
 message(STATUS "GIT_BRANCH_NAME = ${GIT_BRANCH_NAME}")
-if(NOT GIT_BRANCH_NAME STREQUAL "")
+if (NOT GIT_BRANCH_NAME STREQUAL "")
     string(REGEX REPLACE "\n" "" GIT_BRANCH_NAME ${GIT_BRANCH_NAME})
-endif()
+endif ()
 
 set(MILVUS_VERSION "${GIT_BRANCH_NAME}")
 string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]" MILVUS_VERSION "${MILVUS_VERSION}")
 
-find_package(ClangTools)
-set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support")
-
-if(CMAKE_BUILD_TYPE STREQUAL "Release")
+if (CMAKE_BUILD_TYPE STREQUAL "Release")
     set(BUILD_TYPE "Release")
-else()
+else ()
     set(BUILD_TYPE "Debug")
-endif()
+endif ()
 message(STATUS "Build type = ${BUILD_TYPE}")
 
 project(milvus VERSION "${MILVUS_VERSION}")
-project(milvus_engine LANGUAGES CUDA CXX)
+project(milvus_engine LANGUAGES CXX)
 
 unset(CMAKE_EXPORT_COMPILE_COMMANDS CACHE)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
@@ -67,15 +62,15 @@ set(MILVUS_VERSION_MAJOR "${milvus_VERSION_MAJOR}")
 set(MILVUS_VERSION_MINOR "${milvus_VERSION_MINOR}")
 set(MILVUS_VERSION_PATCH "${milvus_VERSION_PATCH}")
 
-if(MILVUS_VERSION_MAJOR STREQUAL ""
+if (MILVUS_VERSION_MAJOR STREQUAL ""
         OR MILVUS_VERSION_MINOR STREQUAL ""
         OR MILVUS_VERSION_PATCH STREQUAL "")
     message(WARNING "Failed to determine Milvus version from git branch name")
-    set(MILVUS_VERSION "0.5.3")
+    set(MILVUS_VERSION "0.6.0")
 endif()
 
 message(STATUS "Build version = ${MILVUS_VERSION}")
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/src/version.h.macro ${CMAKE_CURRENT_SOURCE_DIR}/src/version.h)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/src/version.h.in ${CMAKE_CURRENT_SOURCE_DIR}/src/version.h @ONLY)
 
 message(STATUS "Milvus version: "
         "${MILVUS_VERSION_MAJOR}.${MILVUS_VERSION_MINOR}.${MILVUS_VERSION_PATCH} "
@@ -84,77 +79,111 @@ message(STATUS "Milvus version: "
 set(CMAKE_CXX_STANDARD 14)
 set(CMAKE_CXX_STANDARD_REQUIRED on)
 
-if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
+if (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
     message(STATUS "Building milvus_engine on x86 architecture")
     set(MILVUS_BUILD_ARCH x86_64)
-elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(ppc)")
+elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "(ppc)")
     message(STATUS "Building milvus_engine on ppc architecture")
     set(MILVUS_BUILD_ARCH ppc64le)
-else()
+else ()
     message(WARNING "Unknown processor type")
     message(WARNING "CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}")
     set(MILVUS_BUILD_ARCH unknown)
-endif()
-
-find_package (Python COMPONENTS Interpreter Development)
-
-find_package(CUDA)
-set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES --expt-extended-lambda")
-
-if(CMAKE_BUILD_TYPE STREQUAL "Release")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp")
-    set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3")
-else()
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp")
-    set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g")
-endif()
+endif ()
 
 # Ensure that a default make is set
-if("${MAKE}" STREQUAL "")
-    if(NOT MSVC)
+if ("${MAKE}" STREQUAL "")
+    if (NOT MSVC)
         find_program(MAKE make)
-    endif()
-endif()
+    endif ()
+endif ()
 
 find_path(MYSQL_INCLUDE_DIR
-            NAMES "mysql.h"
-            PATH_SUFFIXES "mysql")
+        NAMES "mysql.h"
+        PATH_SUFFIXES "mysql")
 if (${MYSQL_INCLUDE_DIR} STREQUAL "MYSQL_INCLUDE_DIR-NOTFOUND")
     message(FATAL_ERROR "Could not found MySQL include directory")
-else()
+else ()
     include_directories(${MYSQL_INCLUDE_DIR})
-endif()
+endif ()
 
 set(MILVUS_SOURCE_DIR ${PROJECT_SOURCE_DIR})
 set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR})
 set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src)
+set(MILVUS_THIRDPARTY_SRC ${PROJECT_SOURCE_DIR}/thirdparty)
 
 include(ExternalProject)
 include(DefineOptions)
 include(BuildUtils)
 include(ThirdPartyPackages)
 
-config_summary()
+if (MILVUS_USE_CCACHE)
+    find_program(CCACHE_FOUND ccache)
+    if (CCACHE_FOUND)
+        message(STATUS "Using ccache: ${CCACHE_FOUND}")
+        set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND})
+        set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND})
+        # let ccache preserve C++ comments, because some of them may be
+        # meaningful to the compiler
+        set(ENV{CCACHE_COMMENTS} "1")
+    endif (CCACHE_FOUND)
+endif ()
+
+set(MILVUS_CPU_VERSION false)
+if (MILVUS_GPU_VERSION)
+    message(STATUS "Building Milvus GPU version")
+    add_compile_definitions("MILVUS_GPU_VERSION")
+    enable_language(CUDA)
+    find_package(CUDA 10 REQUIRED)
+    set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES --expt-extended-lambda")
+else ()
+    message(STATUS "Building Milvus CPU version")
+    set(MILVUS_CPU_VERSION true)
+    add_compile_definitions("MILVUS_CPU_VERSION")
+endif ()
+
+if (MILVUS_WITH_PROMETHEUS)
+    add_compile_definitions("MILVUS_WITH_PROMETHEUS")
+endif ()
+
+if (CMAKE_BUILD_TYPE STREQUAL "Release")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp")
+    if (MILVUS_GPU_VERSION)
+        set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3")
+    endif ()
+else ()
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp")
+    if (MILVUS_GPU_VERSION)
+        set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g")
+    endif ()
+endif ()
 
 if (CUSTOMIZATION)
     add_definitions(-DCUSTOMIZATION)
 endif (CUSTOMIZATION)
 
+config_summary()
 add_subdirectory(src)
 
 if (BUILD_UNIT_TEST STREQUAL "ON")
     if (BUILD_COVERAGE STREQUAL "ON")
         set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
-    endif()
+    endif ()
     add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/unittest)
-endif()
+endif ()
 
 add_custom_target(Clean-All COMMAND ${CMAKE_BUILD_TOOL} clean)
 
-if("${MILVUS_DB_PATH}" STREQUAL "")
+if ("${MILVUS_DB_PATH}" STREQUAL "")
     set(MILVUS_DB_PATH "/tmp/milvus")
-endif()
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml)
+endif ()
+
+if (MILVUS_GPU_VERSION)
+    configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_gpu_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml)
+else ()
+    configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_cpu_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml)
+endif ()
+
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.conf)
 
 install(DIRECTORY scripts/
@@ -169,19 +198,22 @@ install(FILES
         DESTINATION
         conf)
 
+find_package(Python COMPONENTS Interpreter Development)
+find_package(ClangTools)
+set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support")
 
 #
 # "make lint" target
 #
-if(NOT MILVUS_VERBOSE_LINT)
-  set(MILVUS_LINT_QUIET "--quiet")
-endif()
+if (NOT MILVUS_VERBOSE_LINT)
+    set(MILVUS_LINT_QUIET "--quiet")
+endif ()
 
-if(NOT LINT_EXCLUSIONS_FILE)
-  # source files matching a glob from a line in this file
-  # will be excluded from linting (cpplint, clang-tidy, clang-format)
-  set(LINT_EXCLUSIONS_FILE ${BUILD_SUPPORT_DIR}/lint_exclusions.txt)
-endif()
+if (NOT LINT_EXCLUSIONS_FILE)
+    # source files matching a glob from a line in this file
+    # will be excluded from linting (cpplint, clang-tidy, clang-format)
+    set(LINT_EXCLUSIONS_FILE ${BUILD_SUPPORT_DIR}/lint_exclusions.txt)
+endif ()
 
 find_program(CPPLINT_BIN NAMES cpplint cpplint.py HINTS ${BUILD_SUPPORT_DIR})
 message(STATUS "Found cpplint executable at ${CPPLINT_BIN}")
@@ -190,77 +222,76 @@ message(STATUS "Found cpplint executable at ${CPPLINT_BIN}")
 # "make lint" targets
 #
 add_custom_target(lint
-                  ${PYTHON_EXECUTABLE}
-                  ${BUILD_SUPPORT_DIR}/run_cpplint.py
-                  --cpplint_binary
-                  ${CPPLINT_BIN}
-                  --exclude_globs
-                  ${LINT_EXCLUSIONS_FILE}
-                  --source_dir
-                  ${CMAKE_CURRENT_SOURCE_DIR}
-                  ${MILVUS_LINT_QUIET})
+        ${PYTHON_EXECUTABLE}
+        ${BUILD_SUPPORT_DIR}/run_cpplint.py
+        --cpplint_binary
+        ${CPPLINT_BIN}
+        --exclude_globs
+        ${LINT_EXCLUSIONS_FILE}
+        --source_dir
+        ${CMAKE_CURRENT_SOURCE_DIR}
+        ${MILVUS_LINT_QUIET})
 
 #
 # "make clang-format" and "make check-clang-format" targets
 #
-if(${CLANG_FORMAT_FOUND})
-  # runs clang format and updates files in place.
-  add_custom_target(clang-format
-                    ${PYTHON_EXECUTABLE}
-                    ${BUILD_SUPPORT_DIR}/run_clang_format.py
-                    --clang_format_binary
-                    ${CLANG_FORMAT_BIN}
-                    --exclude_globs
-                    ${LINT_EXCLUSIONS_FILE}
-                    --source_dir
-                    ${CMAKE_CURRENT_SOURCE_DIR}/src
-                    --fix
-                    ${MILVUS_LINT_QUIET})
-
-  # runs clang format and exits with a non-zero exit code if any files need to be reformatted
-  add_custom_target(check-clang-format
-                    ${PYTHON_EXECUTABLE}
-                    ${BUILD_SUPPORT_DIR}/run_clang_format.py
-                    --clang_format_binary
-                    ${CLANG_FORMAT_BIN}
-                    --exclude_globs
-                    ${LINT_EXCLUSIONS_FILE}
-                    --source_dir
-                    ${CMAKE_CURRENT_SOURCE_DIR}/src
-                    ${MILVUS_LINT_QUIET})
-endif()
+if (${CLANG_FORMAT_FOUND})
+    # runs clang format and updates files in place.
+    add_custom_target(clang-format
+            ${PYTHON_EXECUTABLE}
+            ${BUILD_SUPPORT_DIR}/run_clang_format.py
+            --clang_format_binary
+            ${CLANG_FORMAT_BIN}
+            --exclude_globs
+            ${LINT_EXCLUSIONS_FILE}
+            --source_dir
+            ${CMAKE_CURRENT_SOURCE_DIR}/src
+            --fix
+            ${MILVUS_LINT_QUIET})
+
+    # runs clang format and exits with a non-zero exit code if any files need to be reformatted
+    add_custom_target(check-clang-format
+            ${PYTHON_EXECUTABLE}
+            ${BUILD_SUPPORT_DIR}/run_clang_format.py
+            --clang_format_binary
+            ${CLANG_FORMAT_BIN}
+            --exclude_globs
+            ${LINT_EXCLUSIONS_FILE}
+            --source_dir
+            ${CMAKE_CURRENT_SOURCE_DIR}/src
+            ${MILVUS_LINT_QUIET})
+endif ()
 
 #
 # "make clang-tidy" and "make check-clang-tidy" targets
 #
-if(${CLANG_TIDY_FOUND})
-  # runs clang-tidy and attempts to fix any warning automatically
-  add_custom_target(clang-tidy
-                    ${PYTHON_EXECUTABLE}
-                    ${BUILD_SUPPORT_DIR}/run_clang_tidy.py
-                    --clang_tidy_binary
-                    ${CLANG_TIDY_BIN}
-                    --exclude_globs
-                    ${LINT_EXCLUSIONS_FILE}
-                    --compile_commands
-                    ${CMAKE_BINARY_DIR}/compile_commands.json
-                    --source_dir
-                    ${CMAKE_CURRENT_SOURCE_DIR}/src
-                    --fix
-                    ${MILVUS_LINT_QUIET})
-
-  # runs clang-tidy and exits with a non-zero exit code if any errors are found.
-  add_custom_target(check-clang-tidy
-                    ${PYTHON_EXECUTABLE}
-                    ${BUILD_SUPPORT_DIR}/run_clang_tidy.py
-                    --clang_tidy_binary
-                    ${CLANG_TIDY_BIN}
-                    --exclude_globs
-                    ${LINT_EXCLUSIONS_FILE}
-                    --compile_commands
-                    ${CMAKE_BINARY_DIR}/compile_commands.json
-                    --source_dir
-                    ${CMAKE_CURRENT_SOURCE_DIR}/src
-                    ${MILVUS_LINT_QUIET})
-endif()
-
+if (${CLANG_TIDY_FOUND})
+    # runs clang-tidy and attempts to fix any warning automatically
+    add_custom_target(clang-tidy
+            ${PYTHON_EXECUTABLE}
+            ${BUILD_SUPPORT_DIR}/run_clang_tidy.py
+            --clang_tidy_binary
+            ${CLANG_TIDY_BIN}
+            --exclude_globs
+            ${LINT_EXCLUSIONS_FILE}
+            --compile_commands
+            ${CMAKE_BINARY_DIR}/compile_commands.json
+            --source_dir
+            ${CMAKE_CURRENT_SOURCE_DIR}/src
+            --fix
+            ${MILVUS_LINT_QUIET})
+
+    # runs clang-tidy and exits with a non-zero exit code if any errors are found.
+    add_custom_target(check-clang-tidy
+            ${PYTHON_EXECUTABLE}
+            ${BUILD_SUPPORT_DIR}/run_clang_tidy.py
+            --clang_tidy_binary
+            ${CLANG_TIDY_BIN}
+            --exclude_globs
+            ${LINT_EXCLUSIONS_FILE}
+            --compile_commands
+            ${CMAKE_BINARY_DIR}/compile_commands.json
+            --source_dir
+            ${CMAKE_CURRENT_SOURCE_DIR}/src
+            ${MILVUS_LINT_QUIET})
+endif ()

core/build-support/lint_exclusions.txt

@@ -6,5 +6,5 @@
 *easylogging++*
 *SqliteMetaImpl.cpp
 *src/grpc*
-*src/external*
+*thirdparty*
 *milvus/include*
\ No newline at end of file

core/build.sh

@@ -12,124 +12,143 @@ USE_JFROG_CACHE="OFF"
 RUN_CPPLINT="OFF"
 CUSTOMIZATION="OFF" # default use ori faiss
 CUDA_COMPILER=/usr/local/cuda/bin/nvcc
+GPU_VERSION="OFF" #defaults to CPU version
+WITH_MKL="OFF"
+FAISS_ROOT="" #FAISS root path
+FAISS_SOURCE="BUNDLED"
+WITH_PROMETHEUS="ON"
 
-CUSTOMIZED_FAISS_URL="${FAISS_URL:-NONE}"
-wget -q --method HEAD ${CUSTOMIZED_FAISS_URL}
-if [ $? -eq 0 ]; then
-  CUSTOMIZATION="ON"
-else
-  CUSTOMIZATION="OFF"
-fi
-
-while getopts "p:d:t:ulrcgjhx" arg
-do
-        case $arg in
-             p)
-                INSTALL_PREFIX=$OPTARG
-                ;;
-             d)
-                DB_PATH=$OPTARG
-                ;;
-             t)
-                BUILD_TYPE=$OPTARG # BUILD_TYPE
-                ;;
-             u)
-                echo "Build and run unittest cases" ;
-                BUILD_UNITTEST="ON";
-                ;;
-             l)
-                RUN_CPPLINT="ON"
-                ;;
-             r)
-                if [[ -d ${BUILD_OUTPUT_DIR} ]]; then
-                    rm ./${BUILD_OUTPUT_DIR} -r
-                    MAKE_CLEAN="ON"
-                fi
-                ;;
-             c)
-                BUILD_COVERAGE="ON"
-                ;;
-             g)
-                PROFILING="ON"
-                ;;
-             j)
-                USE_JFROG_CACHE="ON"
-                ;;
-             x)
-                CUSTOMIZATION="OFF" # force use ori faiss
-                ;;
-             h) # help
-                echo "
+while getopts "p:d:t:f:ulrcgjhxzme" arg; do
+  case $arg in
+  p)
+    INSTALL_PREFIX=$OPTARG
+    ;;
+  d)
+    DB_PATH=$OPTARG
+    ;;
+  t)
+    BUILD_TYPE=$OPTARG # BUILD_TYPE
+    ;;
+  f)
+    FAISS_ROOT=$OPTARG
+    FAISS_SOURCE="AUTO"
+    ;;
+  u)
+    echo "Build and run unittest cases"
+    BUILD_UNITTEST="ON"
+    ;;
+  l)
+    RUN_CPPLINT="ON"
+    ;;
+  r)
+    if [[ -d ${BUILD_OUTPUT_DIR} ]]; then
+      rm ./${BUILD_OUTPUT_DIR} -r
+      MAKE_CLEAN="ON"
+    fi
+    ;;
+  c)
+    BUILD_COVERAGE="ON"
+    ;;
+  z)
+    PROFILING="ON"
+    ;;
+  j)
+    USE_JFROG_CACHE="ON"
+    ;;
+  x)
+    CUSTOMIZATION="OFF" # force use ori faiss
+    ;;
+  g)
+    GPU_VERSION="ON"
+    ;;
+  m)
+    WITH_MKL="ON"
+    ;;
+  e)
+    WITH_PROMETHEUS="OFF"
+    ;;
+  h) # help
+    echo "
 
 parameter:
 -p: install prefix(default: $(pwd)/milvus)
 -d: db data path(default: /tmp/milvus)
 -t: build type(default: Debug)
+-f: FAISS root path(default: empty). The path should be an absolute path
+    containing the pre-installed lib/ and include/ directory of FAISS. If they can't be found,
+    we will build the original FAISS from source instead.
 -u: building unit test options(default: OFF)
 -l: run cpplint, clang-format and clang-tidy(default: OFF)
 -r: remove previous build directory(default: OFF)
 -c: code coverage(default: OFF)
--g: profiling(default: OFF)
+-z: profiling(default: OFF)
 -j: use jfrog cache build directory(default: OFF)
+-g: build GPU version(default: OFF)
+-m: build with MKL(default: OFF)
+-e: build without prometheus(default: OFF)
 -h: help
 
 usage:
-./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} [-u] [-l] [-r] [-c] [-g] [-j] [-h]
+./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -f \${FAISS_ROOT} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-e] [-h]
                 "
-                exit 0
-                ;;
-             ?)
-                echo "ERROR! unknown argument"
-        exit 1
-        ;;
-        esac
+    exit 0
+    ;;
+  ?)
+    echo "ERROR! unknown argument"
+    exit 1
+    ;;
+  esac
 done
 
 if [[ ! -d ${BUILD_OUTPUT_DIR} ]]; then
-    mkdir ${BUILD_OUTPUT_DIR}
+  mkdir ${BUILD_OUTPUT_DIR}
 fi
 
 cd ${BUILD_OUTPUT_DIR}
 
 # remove make cache since build.sh -l use default variables
 # force update the variables each time
-make rebuild_cache > /dev/null 2>&1
+make rebuild_cache >/dev/null 2>&1
 
 CMAKE_CMD="cmake \
 -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
 -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}
 -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
+-DFAISS_ROOT=${FAISS_ROOT} \
+-DFAISS_SOURCE=${FAISS_SOURCE} \
 -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
 -DBUILD_COVERAGE=${BUILD_COVERAGE} \
 -DMILVUS_DB_PATH=${DB_PATH} \
 -DMILVUS_ENABLE_PROFILING=${PROFILING} \
 -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
 -DCUSTOMIZATION=${CUSTOMIZATION} \
--DFAISS_URL=${CUSTOMIZED_FAISS_URL} \
+-DMILVUS_GPU_VERSION=${GPU_VERSION} \
+-DFAISS_WITH_MKL=${WITH_MKL} \
+-DMILVUS_WITH_PROMETHEUS=${WITH_PROMETHEUS} \
 ../"
 echo ${CMAKE_CMD}
 ${CMAKE_CMD}
 
 if [[ ${MAKE_CLEAN} == "ON" ]]; then
-    make clean
+  make clean
 fi
 
 if [[ ${RUN_CPPLINT} == "ON" ]]; then
-    # cpplint check
-    make lint
-    if [ $? -ne 0 ]; then
-        echo "ERROR! cpplint check failed"
-        exit 1
-    fi
-    echo "cpplint check passed!"
+  # cpplint check
+  make lint
+  if [ $? -ne 0 ]; then
+    echo "ERROR! cpplint check failed"
+    exit 1
+  fi
+  echo "cpplint check passed!"
 
-    # clang-format check
-    make check-clang-format
-    if [ $? -ne 0 ]; then
-        echo "ERROR! clang-format check failed"
-        exit 1
-    fi
-    echo "clang-format check passed!"
+  # clang-format check
+  make check-clang-format
+  if [ $? -ne 0 ]; then
+    echo "ERROR! clang-format check failed"
+    exit 1
+  fi
+  echo "clang-format check passed!"
 
 #    # clang-tidy check
 #    make check-clang-tidy
@@ -140,11 +159,11 @@ if [[ ${RUN_CPPLINT} == "ON" ]]; then
 #    echo "clang-tidy check passed!"
 else
 
-    # strip binary symbol
-    if [[ ${BUILD_TYPE} != "Debug" ]]; then
-        strip src/milvus_server
-    fi
+  # strip binary symbol
+  if [[ ${BUILD_TYPE} != "Debug" ]]; then
+    strip src/milvus_server
+  fi
 
-    # compile and build
-    make -j 8 install || exit 1
-fi
\ No newline at end of file
+  # compile and build
+  make -j 8 install || exit 1
+fi

core/cmake/DefineOptions.cmake

@@ -13,16 +13,16 @@ macro(define_option name description default)
 endmacro()
 
 function(list_join lst glue out)
-    if("${${lst}}" STREQUAL "")
+    if ("${${lst}}" STREQUAL "")
         set(${out} "" PARENT_SCOPE)
         return()
-    endif()
+    endif ()
 
     list(GET ${lst} 0 joined)
     list(REMOVE_AT ${lst} 0)
-    foreach(item ${${lst}})
+    foreach (item ${${lst}})
         set(joined "${joined}${glue}${item}")
-    endforeach()
+    endforeach ()
     set(${out} ${joined} PARENT_SCOPE)
 endfunction()
 
@@ -35,22 +35,29 @@ macro(define_option_string name description default)
 
     set("${name}_OPTION_ENUM" ${ARGN})
     list_join("${name}_OPTION_ENUM" "|" "${name}_OPTION_ENUM")
-    if(NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
+    if (NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
         set_property(CACHE ${name} PROPERTY STRINGS ${ARGN})
-    endif()
+    endif ()
 endmacro()
 
+#----------------------------------------------------------------------
+set_option_category("GPU version")
+
+define_option(MILVUS_GPU_VERSION "Build GPU version" OFF)
+
 #----------------------------------------------------------------------
 set_option_category("Thirdparty")
 
-set(MILVUS_DEPENDENCY_SOURCE_DEFAULT "AUTO")
+set(MILVUS_DEPENDENCY_SOURCE_DEFAULT "BUNDLED")
 
 define_option_string(MILVUS_DEPENDENCY_SOURCE
-                    "Method to use for acquiring MILVUS's build dependencies"
-                    "${MILVUS_DEPENDENCY_SOURCE_DEFAULT}"
-                    "AUTO"
-                    "BUNDLED"
-                    "SYSTEM")
+        "Method to use for acquiring MILVUS's build dependencies"
+        "${MILVUS_DEPENDENCY_SOURCE_DEFAULT}"
+        "AUTO"
+        "BUNDLED"
+        "SYSTEM")
+
+define_option(MILVUS_USE_CCACHE "Use ccache when compiling (if available)" ON)
 
 define_option(MILVUS_VERBOSE_THIRDPARTY_BUILD
         "Show output from ExternalProjects rather than just logging to files" ON)
@@ -70,33 +77,21 @@ define_option(MILVUS_WITH_YAMLCPP "Build with yaml-cpp library" ON)
 if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
     define_option(MILVUS_WITH_LIBUNWIND "Build with libunwind" ON)
     define_option(MILVUS_WITH_GPERFTOOLS "Build with gperftools" ON)
-endif()
+endif ()
 
 define_option(MILVUS_WITH_GRPC "Build with GRPC" ON)
 
 define_option(MILVUS_WITH_ZLIB "Build with zlib compression" ON)
 
-#----------------------------------------------------------------------
-if(MSVC)
-    set_option_category("MSVC")
-
-    define_option(MSVC_LINK_VERBOSE
-            "Pass verbose linking options when linking libraries and executables"
-            OFF)
-
-    define_option(MILVUS_USE_STATIC_CRT "Build MILVUS with statically linked CRT" OFF)
-endif()
-
-
 #----------------------------------------------------------------------
 set_option_category("Test and benchmark")
 
 unset(MILVUS_BUILD_TESTS CACHE)
 if (BUILD_UNIT_TEST)
     define_option(MILVUS_BUILD_TESTS "Build the MILVUS googletest unit tests" ON)
-else()
+else ()
     define_option(MILVUS_BUILD_TESTS "Build the MILVUS googletest unit tests" OFF)
-endif(BUILD_UNIT_TEST)
+endif (BUILD_UNIT_TEST)
 
 #----------------------------------------------------------------------
 macro(config_summary)
@@ -108,12 +103,12 @@ macro(config_summary)
     message(STATUS "  Generator: ${CMAKE_GENERATOR}")
     message(STATUS "  Build type: ${CMAKE_BUILD_TYPE}")
     message(STATUS "  Source directory: ${CMAKE_CURRENT_SOURCE_DIR}")
-    if(${CMAKE_EXPORT_COMPILE_COMMANDS})
+    if (${CMAKE_EXPORT_COMPILE_COMMANDS})
         message(
                 STATUS "  Compile commands: ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json")
-    endif()
+    endif ()
 
-    foreach(category ${MILVUS_OPTION_CATEGORIES})
+    foreach (category ${MILVUS_OPTION_CATEGORIES})
 
         message(STATUS)
         message(STATUS "${category} options:")
@@ -121,50 +116,50 @@ macro(config_summary)
         set(option_names ${MILVUS_${category}_OPTION_NAMES})
 
         set(max_value_length 0)
-        foreach(name ${option_names})
+        foreach (name ${option_names})
             string(LENGTH "\"${${name}}\"" value_length)
-            if(${max_value_length} LESS ${value_length})
+            if (${max_value_length} LESS ${value_length})
                 set(max_value_length ${value_length})
-            endif()
-        endforeach()
+            endif ()
+        endforeach ()
 
-        foreach(name ${option_names})
-            if("${${name}_OPTION_TYPE}" STREQUAL "string")
+        foreach (name ${option_names})
+            if ("${${name}_OPTION_TYPE}" STREQUAL "string")
                 set(value "\"${${name}}\"")
-            else()
+            else ()
                 set(value "${${name}}")
-            endif()
+            endif ()
 
             set(default ${${name}_OPTION_DEFAULT})
             set(description ${${name}_OPTION_DESCRIPTION})
             string(LENGTH ${description} description_length)
-            if(${description_length} LESS 70)
+            if (${description_length} LESS 70)
                 string(
                         SUBSTRING
                         "                                                                     "
                         ${description_length} -1 description_padding)
-            else()
+            else ()
                 set(description_padding "
                 ")
-            endif()
+            endif ()
 
             set(comment "[${name}]")
 
-            if("${value}" STREQUAL "${default}")
+            if ("${value}" STREQUAL "${default}")
                 set(comment "[default] ${comment}")
-            endif()
+            endif ()
 
-            if(NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
+            if (NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
                 set(comment "${comment} [${${name}_OPTION_ENUM}]")
-            endif()
+            endif ()
 
             string(
                     SUBSTRING "${value}                                                             "
                     0 ${max_value_length} value)
 
             message(STATUS "  ${description} ${description_padding} ${value} ${comment}")
-        endforeach()
+        endforeach ()
 
-    endforeach()
+    endforeach ()
 
 endmacro()

core/cmake/ThirdPartyPackages.cmake

@@ -164,8 +164,10 @@ endif ()
 
 macro(resolve_dependency DEPENDENCY_NAME)
     if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO")
-        #disable find_package for now
-        build_dependency(${DEPENDENCY_NAME})
+        find_package(${DEPENDENCY_NAME} MODULE)
+        if(NOT ${${DEPENDENCY_NAME}_FOUND})
+          build_dependency(${DEPENDENCY_NAME})
+        endif()
     elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED")
         build_dependency(${DEPENDENCY_NAME})
     elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM")

core/conf/server_config.template → core/conf/server_cpu_config.template

@@ -27,10 +27,7 @@ metric_config:
     port: 8080                      # port prometheus uses to fetch metrics, must in range [1025, 65534]
 
 cache_config:
-  cpu_cache_capacity: 16            # GB, CPU memory used for cache, must be a positive integer
-  cpu_cache_threshold: 0.85         # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0]
-  gpu_cache_capacity: 4             # GB, GPU memory used for cache, must be a positive integer
-  gpu_cache_threshold: 0.85         # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0]
+  cpu_cache_capacity: 16            # GB, size of CPU memory used for cache, must be a positive integer
   cache_insert_data: false          # whether to load inserted data into cache, must be a boolean
 
 engine_config:
@@ -38,8 +35,10 @@ engine_config:
                                     # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times
   gpu_search_threshold: 1000        # threshold beyond which the search computation is executed on GPUs only
 
-resource_config:
-  search_resources:                 # define the devices used for search computation, must be in format: cpu or gpux
-    - cpu
+gpu_resource_config:
+  enable: false                     # whether to enable GPU resources
+  cache_capacity: 4                 # GB, size of GPU memory per card used for cache, must be a positive integer
+  search_resources:                 # define the GPU devices used for search computation, must be in format gpux
+    - gpu0
+  build_index_resources:            # define the GPU devices used for index building, must be in format gpux
     - gpu0
-  index_build_device: gpu0          # GPU used for building index, must be in format: gpux
\ No newline at end of file

core/conf/server_gpu_config.template

+# Default values are used when you make no changes to the following parameters.
+
+server_config:
+  address: 0.0.0.0                  # milvus server ip address (IPv4)
+  port: 19530                       # milvus server port, must in range [1025, 65534]
+  deploy_mode: single               # deployment type: single, cluster_readonly, cluster_writable
+  time_zone: UTC+8                  # time zone, must be in format: UTC+X
+
+db_config:
+  primary_path: @MILVUS_DB_PATH@    # path used to store data and meta
+  secondary_path:                   # path used to store data only, split by semicolon
+
+  backend_url: sqlite://:@:/        # URI format: dialect://username:password@host:port/database
+                                    # Keep 'dialect://:@:/', and replace other texts with real values
+                                    # Replace 'dialect' with 'mysql' or 'sqlite'
+
+  insert_buffer_size: 4             # GB, maximum insert buffer size allowed, must be a positive integer
+                                    # sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory
+
+  preload_table:                    # preload data at startup, '*' means load all tables, empty value means no preload
+                                    # you can specify preload tables like this: table1,table2,table3
+
+metric_config:
+  enable_monitor: false             # enable monitoring or not, must be a boolean
+  collector: prometheus             # prometheus
+  prometheus_config:
+    port: 8080                      # port prometheus uses to fetch metrics, must in range [1025, 65534]
+
+cache_config:
+  cpu_cache_capacity: 16            # GB, size of CPU memory used for cache, must be a positive integer
+  cache_insert_data: false          # whether to load inserted data into cache, must be a boolean
+
+engine_config:
+  use_blas_threshold: 1100          # if nq <  use_blas_threshold, use SSE, faster with fluctuated response times
+                                    # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times
+  gpu_search_threshold: 1000        # threshold beyond which the search computation is executed on GPUs only
+
+gpu_resource_config:
+  enable: true                      # whether to enable GPU resources
+  cache_capacity: 4                 # GB, size of GPU memory per card used for cache, must be a positive integer
+  search_resources:                 # define the GPU devices used for search computation, must be in format gpux
+    - gpu0
+  build_index_resources:            # define the GPU devices used for index building, must be in format gpux
+    - gpu0

core/coverage.sh

@@ -122,9 +122,7 @@ ${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \
     "*/src/server/Server.cpp" \
     "*/src/server/DBWrapper.cpp" \
     "*/src/server/grpc_impl/GrpcServer.cpp" \
-    "*/easylogging++.h" \
-    "*/easylogging++.cc" \
-    "*/src/external/*"
+    "*/thirdparty/*"
 
 if [ $? -ne 0 ]; then
     echo "generate ${FILE_INFO_OUTPUT_NEW} failed"

core/scripts/migration/README.md

+## Data Migration
+
+####0.3.x
+legacy data is not migrate-able for later versions
+
+####0.4.x
+legacy data can be reused directly by 0.5.x
+
+legacy data can be migrated to 0.6.x
+
+####0.5.x
+legacy data can be migrated to 0.6.x
+
+####0.6.x
+how to migrate legacy 0.4.x/0.5.x data
+
+for sqlite meta:
+```shell
+ $ sqlite3 [parth_to]/meta.sqlite < sqlite_4_to_6.sql
+```
+
+for mysql meta:
+```shell
+ $ mysql -h127.0.0.1 -uroot -p123456 -Dmilvus < mysql_4_to_6.sql
+```
+
+
+

core/scripts/migration/mysql_4_to_6.sql

+alter table Tables add column owner_table VARCHAR(255) DEFAULT '' NOT NULL;
+alter table Tables add column partition_tag VARCHAR(255) DEFAULT '' NOT NULL;
+alter table Tables add column version VARCHAR(64) DEFAULT '0.6.0' NOT NULL;
+update Tables set version='0.6.0';

core/scripts/migration/mysql_6_to_4.sql

+alter table Tables drop column owner_table;
+alter table Tables drop column partition_tag;
+alter table Tables drop column version;

core/scripts/migration/sqlite_4_to_6.sql

+alter table Tables add column 'owner_table' TEXT DEFAULT '' NOT NULL;
+alter table Tables add column 'partition_tag' TEXT DEFAULT '' NOT NULL;
+alter table Tables add column 'version' TEXT DEFAULT '0.6.0' NOT NULL;
+update Tables set version='0.6.0';

core/scripts/migration/sqlite_6_to_4.sql

+CREATE TABLE 'TempTables' ( 'id' INTEGER PRIMARY KEY NOT NULL , 'table_id' TEXT UNIQUE NOT NULL , 'state' INTEGER NOT NULL , 'dimension' INTEGER NOT NULL , 'created_on' INTEGER NOT NULL , 'flag' INTEGER DEFAULT 0 NOT NULL , 'index_file_size' INTEGER NOT NULL , 'engine_type' INTEGER NOT NULL , 'nlist' INTEGER NOT NULL , 'metric_type' INTEGER NOT NULL);
+
+INSERT INTO TempTables SELECT id, table_id, state, dimension, created_on, flag, index_file_size, engine_type, nlist, metric_type FROM Tables;
+
+DROP TABLE Tables;
+
+ALTER TABLE TempTables RENAME TO Tables;

core/src/CMakeLists.txt

@@ -19,13 +19,15 @@
 
 include_directories(${MILVUS_SOURCE_DIR})
 include_directories(${MILVUS_ENGINE_SRC})
+include_directories(${MILVUS_THIRDPARTY_SRC})
 
-include_directories(${CUDA_TOOLKIT_ROOT_DIR}/include)
 include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-status)
 include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-milvus)
 
-#this statement must put here, since the INDEX_INCLUDE_DIRS is defined in code/CMakeList.txt
 add_subdirectory(index)
+if (FAISS_WITH_MKL)
+    add_compile_definitions("WITH_MKL")
+endif ()
 
 set(INDEX_INCLUDE_DIRS ${INDEX_INCLUDE_DIRS} PARENT_SCOPE)
 foreach (dir ${INDEX_INCLUDE_DIRS})
@@ -35,6 +37,7 @@ endforeach ()
 aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/metrics metrics_files)
+aux_source_directory(${MILVUS_ENGINE_SRC}/metrics/prometheus metrics_prometheus_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files)
@@ -64,15 +67,21 @@ set(scheduler_files
         ${scheduler_task_files}
         )
 
-aux_source_directory(${MILVUS_ENGINE_SRC}/external/easyloggingpp external_easyloggingpp_files)
-aux_source_directory(${MILVUS_ENGINE_SRC}/external/nlohmann external_nlohmann_files)
-set(external_files
-        ${external_easyloggingpp_files}
-        ${external_nlohmann_files}
+aux_source_directory(${MILVUS_THIRDPARTY_SRC}/easyloggingpp thirdparty_easyloggingpp_files)
+aux_source_directory(${MILVUS_THIRDPARTY_SRC}/nlohmann thirdparty_nlohmann_files)
+set(thirdparty_files
+        ${thirdparty_easyloggingpp_files}
+        ${thirdparty_nlohmann_files}
         )
 
 aux_source_directory(${MILVUS_ENGINE_SRC}/server server_files)
-aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl grpc_server_files)
+aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl/request grpc_request_files)
+aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl grpc_impl_files)
+set(grpc_server_files
+        ${grpc_request_files}
+        ${grpc_impl_files}
+        )
+
 aux_source_directory(${MILVUS_ENGINE_SRC}/utils utils_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_files)
 
@@ -84,11 +93,16 @@ set(engine_files
         ${db_insert_files}
         ${db_meta_files}
         ${metrics_files}
-        ${external_files}
+        ${thirdparty_files}
         ${utils_files}
         ${wrapper_files}
         )
 
+if (MILVUS_WITH_PROMETHEUS)
+    set(engine_files ${engine_files}
+            ${metrics_prometheus_files})
+endif ()
+
 set(client_grpc_lib
         grpcpp_channelz
         grpc++
@@ -109,35 +123,50 @@ set(boost_lib
         libboost_serialization.a
         )
 
-set(cuda_lib
-        ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so
-        cudart
-        cublas
-        )
-
 set(third_party_libs
         sqlite
         ${client_grpc_lib}
         yaml-cpp
-        ${prometheus_lib}
-        ${cuda_lib}
         mysqlpp
         zlib
         ${boost_lib}
         )
 
-if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
+if (MILVUS_GPU_VERSION)
+    include_directories(${CUDA_INCLUDE_DIRS})
+    link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
+    set(cuda_lib
+            ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so
+            cudart
+            cublas
+            )
     set(third_party_libs ${third_party_libs}
-        gperftools
-        libunwind
-        )
+            ${cuda_lib}
+            )
+    aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/gpu wrapper_gpu_files)
+    set(engine_files ${engine_files}
+            ${wrapper_gpu_files}
+            )
+endif ()
+
+if (MILVUS_ENABLE_PROFILING)
+    set(third_party_libs ${third_party_libs}
+            gperftools
+            libunwind
+            )
+endif ()
+
+if (MILVUS_WITH_PROMETHEUS)
+    set(third_party_libs ${third_party_libs}
+            ${prometheus_lib}
+            )
 endif ()
 
-link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
 set(engine_libs
         pthread
         libgomp.a
         libgfortran.a
+        dl
         )
 
 if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
@@ -147,26 +176,33 @@ if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
             )
 endif ()
 
-cuda_add_library(milvus_engine STATIC ${engine_files})
+add_library(milvus_engine STATIC ${engine_files})
 target_link_libraries(milvus_engine
         knowhere
-        ${engine_libs}
         ${third_party_libs}
+        ${engine_libs}
         )
 
-add_library(metrics STATIC ${metrics_files})
+if (MILVUS_WITH_PROMETHEUS)
+    add_library(metrics STATIC ${metrics_files} ${metrics_prometheus_files})
+else ()
+    add_library(metrics STATIC ${metrics_files})
+endif ()
 
 set(metrics_lib
         yaml-cpp
-        ${prometheus_lib}
         )
 
+if (MILVUS_WITH_PROMETHEUS)
+    set(metrics_lib ${metrics_lib}
+            ${prometheus_lib}
+            )
+endif ()
+
 target_link_libraries(metrics ${metrics_lib})
 
 set(server_libs
         milvus_engine
-        pthread
-        dl
         metrics
         )
 

core/src/cache/Cache.inl

@@ -15,24 +15,18 @@
 // specific language governing permissions and limitations
 // under the License.
 
-
-
-
 namespace milvus {
 namespace cache {
 
 constexpr double DEFAULT_THRESHHOLD_PERCENT = 0.85;
 
-template<typename ItemObj>
+template <typename ItemObj>
 Cache<ItemObj>::Cache(int64_t capacity, uint64_t cache_max_count)
-    : usage_(0),
-      capacity_(capacity),
-      freemem_percent_(DEFAULT_THRESHHOLD_PERCENT),
-      lru_(cache_max_count) {
-//    AGENT_LOG_DEBUG << "Construct Cache with capacity " << std::to_string(mem_capacity)
+    : usage_(0), capacity_(capacity), freemem_percent_(DEFAULT_THRESHHOLD_PERCENT), lru_(cache_max_count) {
+    //    AGENT_LOG_DEBUG << "Construct Cache with capacity " << std::to_string(mem_capacity)
 }
 
-template<typename ItemObj>
+template <typename ItemObj>
 void
 Cache<ItemObj>::set_capacity(int64_t capacity) {
     if (capacity > 0) {
@@ -41,23 +35,23 @@ Cache<ItemObj>::set_capacity(int64_t capacity) {
     }
 }
 
-template<typename ItemObj>
+template <typename ItemObj>
 size_t
 Cache<ItemObj>::size() const {
     std::lock_guard<std::mutex> lock(mutex_);
     return lru_.size();
 }
 
-template<typename ItemObj>
+template <typename ItemObj>
 bool
-Cache<ItemObj>::exists(const std::string &key) {
+Cache<ItemObj>::exists(const std::string& key) {
     std::lock_guard<std::mutex> lock(mutex_);
     return lru_.exists(key);
 }
 
-template<typename ItemObj>
+template <typename ItemObj>
 ItemObj
-Cache<ItemObj>::get(const std::string &key) {
+Cache<ItemObj>::get(const std::string& key) {
     std::lock_guard<std::mutex> lock(mutex_);
     if (!lru_.exists(key)) {
         return nullptr;
@@ -66,60 +60,59 @@ Cache<ItemObj>::get(const std::string &key) {
     return lru_.get(key);
 }
 
-template<typename ItemObj>
+template <typename ItemObj>
 void
-Cache<ItemObj>::insert(const std::string &key, const ItemObj &item) {
+Cache<ItemObj>::insert(const std::string& key, const ItemObj& item) {
     if (item == nullptr) {
         return;
     }
 
-//    if(item->size() > capacity_) {
-//        SERVER_LOG_ERROR << "Item size " << item->size()
-//                        << " is too large to insert into cache, capacity " << capacity_;
-//        return;
-//    }
+    //    if(item->size() > capacity_) {
+    //        SERVER_LOG_ERROR << "Item size " << item->size()
+    //                        << " is too large to insert into cache, capacity " << capacity_;
+    //        return;
+    //    }
 
-    //calculate usage
+    // calculate usage
     {
         std::lock_guard<std::mutex> lock(mutex_);
 
-        //if key already exist, subtract old item size
+        // if key already exist, subtract old item size
         if (lru_.exists(key)) {
-            const ItemObj &old_item = lru_.get(key);
+            const ItemObj& old_item = lru_.get(key);
             usage_ -= old_item->Size();
         }
 
-        //plus new item size
+        // plus new item size
         usage_ += item->Size();
     }
 
-    //if usage exceed capacity, free some items
+    // if usage exceed capacity, free some items
     if (usage_ > capacity_) {
-        SERVER_LOG_DEBUG << "Current usage " << usage_
-                         << " exceeds cache capacity " << capacity_
+        SERVER_LOG_DEBUG << "Current usage " << usage_ << " exceeds cache capacity " << capacity_
                          << ", start free memory";
         free_memory();
     }
 
-    //insert new item
+    // insert new item
     {
         std::lock_guard<std::mutex> lock(mutex_);
 
         lru_.put(key, item);
-        SERVER_LOG_DEBUG << "Insert " << key << " size:" << item->Size()
-                         << " bytes into cache, usage: " << usage_ << " bytes";
+        SERVER_LOG_DEBUG << "Insert " << key << " size:" << item->Size() << " bytes into cache, usage: " << usage_
+                         << " bytes";
     }
 }
 
-template<typename ItemObj>
+template <typename ItemObj>
 void
-Cache<ItemObj>::erase(const std::string &key) {
+Cache<ItemObj>::erase(const std::string& key) {
     std::lock_guard<std::mutex> lock(mutex_);
     if (!lru_.exists(key)) {
         return;
     }
 
-    const ItemObj &old_item = lru_.get(key);
+    const ItemObj& old_item = lru_.get(key);
     usage_ -= old_item->Size();
 
     SERVER_LOG_DEBUG << "Erase " << key << " size: " << old_item->Size();
@@ -127,7 +120,7 @@ Cache<ItemObj>::erase(const std::string &key) {
     lru_.erase(key);
 }
 
-template<typename ItemObj>
+template <typename ItemObj>
 void
 Cache<ItemObj>::clear() {
     std::lock_guard<std::mutex> lock(mutex_);
@@ -137,15 +130,16 @@ Cache<ItemObj>::clear() {
 }
 
 /* free memory space when CACHE occupation exceed its capacity */
-template<typename ItemObj>
+template <typename ItemObj>
 void
 Cache<ItemObj>::free_memory() {
-    if (usage_ <= capacity_) return;
+    if (usage_ <= capacity_)
+        return;
 
     int64_t threshhold = capacity_ * freemem_percent_;
     int64_t delta_size = usage_ - threshhold;
     if (delta_size <= 0) {
-        delta_size = 1;//ensure at least one item erased
+        delta_size = 1;  // ensure at least one item erased
     }
 
     std::set<std::string> key_array;
@@ -156,8 +150,8 @@ Cache<ItemObj>::free_memory() {
 
         auto it = lru_.rbegin();
         while (it != lru_.rend() && released_size < delta_size) {
-            auto &key = it->first;
-            auto &obj_ptr = it->second;
+            auto& key = it->first;
+            auto& obj_ptr = it->second;
 
             key_array.emplace(key);
             released_size += obj_ptr->Size();
@@ -167,14 +161,14 @@ Cache<ItemObj>::free_memory() {
 
     SERVER_LOG_DEBUG << "to be released memory size: " << released_size;
 
-    for (auto &key : key_array) {
+    for (auto& key : key_array) {
         erase(key);
     }
 
     print();
 }
 
-template<typename ItemObj>
+template <typename ItemObj>
 void
 Cache<ItemObj>::print() {
     size_t cache_count = 0;
@@ -188,7 +182,5 @@ Cache<ItemObj>::print() {
     SERVER_LOG_DEBUG << "[Cache capacity]: " << capacity_ << " bytes";
 }
 
-} // namespace cache
-} // namespace milvus
-
-
+}  // namespace cache
+}  // namespace milvus

core/src/cache/CacheMgr.inl

@@ -15,21 +15,18 @@
 // specific language governing permissions and limitations
 // under the License.
 
-
-
 namespace milvus {
 namespace cache {
 
-template<typename ItemObj>
+template <typename ItemObj>
 CacheMgr<ItemObj>::CacheMgr() {
 }
 
-template<typename ItemObj>
+template <typename ItemObj>
 CacheMgr<ItemObj>::~CacheMgr() {
-
 }
 
-template<typename ItemObj>
+template <typename ItemObj>
 uint64_t
 CacheMgr<ItemObj>::ItemCount() const {
     if (cache_ == nullptr) {
@@ -37,12 +34,12 @@ CacheMgr<ItemObj>::ItemCount() const {
         return 0;
     }
 
-    return (uint64_t) (cache_->size());
+    return (uint64_t)(cache_->size());
 }
 
-template<typename ItemObj>
+template <typename ItemObj>
 bool
-CacheMgr<ItemObj>::ItemExists(const std::string &key) {
+CacheMgr<ItemObj>::ItemExists(const std::string& key) {
     if (cache_ == nullptr) {
         SERVER_LOG_ERROR << "Cache doesn't exist";
         return false;
@@ -51,9 +48,9 @@ CacheMgr<ItemObj>::ItemExists(const std::string &key) {
     return cache_->exists(key);
 }
 
-template<typename ItemObj>
+template <typename ItemObj>
 ItemObj
-CacheMgr<ItemObj>::GetItem(const std::string &key) {
+CacheMgr<ItemObj>::GetItem(const std::string& key) {
     if (cache_ == nullptr) {
         SERVER_LOG_ERROR << "Cache doesn't exist";
         return nullptr;
@@ -62,9 +59,9 @@ CacheMgr<ItemObj>::GetItem(const std::string &key) {
     return cache_->get(key);
 }
 
-template<typename ItemObj>
+template <typename ItemObj>
 void
-CacheMgr<ItemObj>::InsertItem(const std::string &key, const ItemObj &data) {
+CacheMgr<ItemObj>::InsertItem(const std::string& key, const ItemObj& data) {
     if (cache_ == nullptr) {
         SERVER_LOG_ERROR << "Cache doesn't exist";
         return;
@@ -74,9 +71,9 @@ CacheMgr<ItemObj>::InsertItem(const std::string &key, const ItemObj &data) {
     server::Metrics::GetInstance().CacheAccessTotalIncrement();
 }
 
-template<typename ItemObj>
+template <typename ItemObj>
 void
-CacheMgr<ItemObj>::EraseItem(const std::string &key) {
+CacheMgr<ItemObj>::EraseItem(const std::string& key) {
     if (cache_ == nullptr) {
         SERVER_LOG_ERROR << "Cache doesn't exist";
         return;
@@ -86,7 +83,7 @@ CacheMgr<ItemObj>::EraseItem(const std::string &key) {
     server::Metrics::GetInstance().CacheAccessTotalIncrement();
 }
 
-template<typename ItemObj>
+template <typename ItemObj>
 void
 CacheMgr<ItemObj>::PrintInfo() {
     if (cache_ == nullptr) {
@@ -97,7 +94,7 @@ CacheMgr<ItemObj>::PrintInfo() {
     cache_->print();
 }
 
-template<typename ItemObj>
+template <typename ItemObj>
 void
 CacheMgr<ItemObj>::ClearCache() {
     if (cache_ == nullptr) {
@@ -108,7 +105,7 @@ CacheMgr<ItemObj>::ClearCache() {
     cache_->clear();
 }
 
-template<typename ItemObj>
+template <typename ItemObj>
 int64_t
 CacheMgr<ItemObj>::CacheUsage() const {
     if (cache_ == nullptr) {
@@ -119,7 +116,7 @@ CacheMgr<ItemObj>::CacheUsage() const {
     return cache_->usage();
 }
 
-template<typename ItemObj>
+template <typename ItemObj>
 int64_t
 CacheMgr<ItemObj>::CacheCapacity() const {
     if (cache_ == nullptr) {
@@ -130,7 +127,7 @@ CacheMgr<ItemObj>::CacheCapacity() const {
     return cache_->capacity();
 }
 
-template<typename ItemObj>
+template <typename ItemObj>
 void
 CacheMgr<ItemObj>::SetCapacity(int64_t capacity) {
     if (cache_ == nullptr) {
@@ -140,6 +137,5 @@ CacheMgr<ItemObj>::SetCapacity(int64_t capacity) {
     cache_->set_capacity(capacity);
 }
 
-} // namespace cache
-} // namespace milvus
-
+}  // namespace cache
+}  // namespace milvus

core/src/cache/GpuCacheMgr.cpp

@@ -37,7 +37,7 @@ GpuCacheMgr::GpuCacheMgr() {
     Status s;
 
     int64_t gpu_cache_cap;
-    s = config.GetCacheConfigGpuCacheCapacity(gpu_cache_cap);
+    s = config.GetGpuResourceConfigCacheCapacity(gpu_cache_cap);
     if (!s.ok()) {
         SERVER_LOG_ERROR << s.message();
     }
@@ -45,7 +45,7 @@ GpuCacheMgr::GpuCacheMgr() {
     cache_ = std::make_shared<Cache<DataObjPtr>>(cap, 1UL << 32);
 
     float gpu_mem_threshold;
-    s = config.GetCacheConfigGpuCacheThreshold(gpu_mem_threshold);
+    s = config.GetGpuResourceConfigCacheThreshold(gpu_mem_threshold);
     if (!s.ok()) {
         SERVER_LOG_ERROR << s.message();
     }

core/src/db/DB.h

@@ -47,44 +47,68 @@ class DB {
 
     virtual Status
     CreateTable(meta::TableSchema& table_schema_) = 0;
+
     virtual Status
-    DeleteTable(const std::string& table_id, const meta::DatesT& dates) = 0;
+    DropTable(const std::string& table_id, const meta::DatesT& dates) = 0;
+
     virtual Status
     DescribeTable(meta::TableSchema& table_schema_) = 0;
+
     virtual Status
     HasTable(const std::string& table_id, bool& has_or_not_) = 0;
+
     virtual Status
     AllTables(std::vector<meta::TableSchema>& table_schema_array) = 0;
+
     virtual Status
     GetTableRowCount(const std::string& table_id, uint64_t& row_count) = 0;
+
     virtual Status
     PreloadTable(const std::string& table_id) = 0;
+
     virtual Status
     UpdateTableFlag(const std::string& table_id, int64_t flag) = 0;
 
     virtual Status
-    InsertVectors(const std::string& table_id_, uint64_t n, const float* vectors, IDNumbers& vector_ids_) = 0;
+    CreatePartition(const std::string& table_id, const std::string& partition_name,
+                    const std::string& partition_tag) = 0;
+
+    virtual Status
+    DropPartition(const std::string& partition_name) = 0;
 
     virtual Status
-    Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
-          ResultIds& result_ids, ResultDistances& result_distances) = 0;
+    DropPartitionByTag(const std::string& table_id, const std::string& partition_tag) = 0;
 
     virtual Status
-    Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
-          const meta::DatesT& dates, ResultIds& result_ids, ResultDistances& result_distances) = 0;
+    ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partiton_schema_array) = 0;
 
     virtual Status
-    Query(const std::string& table_id, const std::vector<std::string>& file_ids, uint64_t k, uint64_t nq,
+    InsertVectors(const std::string& table_id, const std::string& partition_tag, uint64_t n, const float* vectors,
+                  IDNumbers& vector_ids_) = 0;
+
+    virtual Status
+    Query(const std::string& table_id, const std::vector<std::string>& partition_tags, uint64_t k, uint64_t nq,
+          uint64_t nprobe, const float* vectors, ResultIds& result_ids, ResultDistances& result_distances) = 0;
+
+    virtual Status
+    Query(const std::string& table_id, const std::vector<std::string>& partition_tags, uint64_t k, uint64_t nq,
           uint64_t nprobe, const float* vectors, const meta::DatesT& dates, ResultIds& result_ids,
           ResultDistances& result_distances) = 0;
 
+    virtual Status
+    QueryByFileID(const std::string& table_id, const std::vector<std::string>& file_ids, uint64_t k, uint64_t nq,
+                  uint64_t nprobe, const float* vectors, const meta::DatesT& dates, ResultIds& result_ids,
+                  ResultDistances& result_distances) = 0;
+
     virtual Status
     Size(uint64_t& result) = 0;
 
     virtual Status
     CreateIndex(const std::string& table_id, const TableIndex& index) = 0;
+
     virtual Status
     DescribeIndex(const std::string& table_id, TableIndex& index) = 0;
+
     virtual Status
     DropIndex(const std::string& table_id) = 0;
 

core/src/db/DBImpl.h

@@ -57,7 +57,7 @@ class DBImpl : public DB {
     CreateTable(meta::TableSchema& table_schema) override;
 
     Status
-    DeleteTable(const std::string& table_id, const meta::DatesT& dates) override;
+    DropTable(const std::string& table_id, const meta::DatesT& dates) override;
 
     Status
     DescribeTable(meta::TableSchema& table_schema) override;
@@ -78,7 +78,21 @@ class DBImpl : public DB {
     GetTableRowCount(const std::string& table_id, uint64_t& row_count) override;
 
     Status
-    InsertVectors(const std::string& table_id, uint64_t n, const float* vectors, IDNumbers& vector_ids) override;
+    CreatePartition(const std::string& table_id, const std::string& partition_name,
+                    const std::string& partition_tag) override;
+
+    Status
+    DropPartition(const std::string& partition_name) override;
+
+    Status
+    DropPartitionByTag(const std::string& table_id, const std::string& partition_tag) override;
+
+    Status
+    ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partiton_schema_array) override;
+
+    Status
+    InsertVectors(const std::string& table_id, const std::string& partition_tag, uint64_t n, const float* vectors,
+                  IDNumbers& vector_ids) override;
 
     Status
     CreateIndex(const std::string& table_id, const TableIndex& index) override;
@@ -90,18 +104,19 @@ class DBImpl : public DB {
     DropIndex(const std::string& table_id) override;
 
     Status
-    Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
-          ResultIds& result_ids, ResultDistances& result_distances) override;
-
-    Status
-    Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
-          const meta::DatesT& dates, ResultIds& result_ids, ResultDistances& result_distances) override;
+    Query(const std::string& table_id, const std::vector<std::string>& partition_tags, uint64_t k, uint64_t nq,
+          uint64_t nprobe, const float* vectors, ResultIds& result_ids, ResultDistances& result_distances) override;
 
     Status
-    Query(const std::string& table_id, const std::vector<std::string>& file_ids, uint64_t k, uint64_t nq,
+    Query(const std::string& table_id, const std::vector<std::string>& partition_tags, uint64_t k, uint64_t nq,
           uint64_t nprobe, const float* vectors, const meta::DatesT& dates, ResultIds& result_ids,
           ResultDistances& result_distances) override;
 
+    Status
+    QueryByFileID(const std::string& table_id, const std::vector<std::string>& file_ids, uint64_t k, uint64_t nq,
+                  uint64_t nprobe, const float* vectors, const meta::DatesT& dates, ResultIds& result_ids,
+                  ResultDistances& result_distances) override;
+
     Status
     Size(uint64_t& result) override;
 
@@ -137,6 +152,29 @@ class DBImpl : public DB {
     Status
     MemSerialize();
 
+    Status
+    GetFilesToSearch(const std::string& table_id, const std::vector<size_t>& file_ids, const meta::DatesT& dates,
+                     meta::TableFilesSchema& files);
+
+    Status
+    GetPartitionsByTags(const std::string& table_id, const std::vector<std::string>& partition_tags,
+                        std::set<std::string>& partition_name_array);
+
+    Status
+    DropTableRecursively(const std::string& table_id, const meta::DatesT& dates);
+
+    Status
+    UpdateTableIndexRecursively(const std::string& table_id, const TableIndex& index);
+
+    Status
+    BuildTableIndexRecursively(const std::string& table_id, const TableIndex& index);
+
+    Status
+    DropTableIndexRecursively(const std::string& table_id);
+
+    Status
+    GetTableRowCountRecursively(const std::string& table_id, uint64_t& row_count);
+
  private:
     const DBOptions options_;
 

core/src/db/Options.h

@@ -33,7 +33,7 @@ static const char* ARCHIVE_CONF_DISK = "disk";
 static const char* ARCHIVE_CONF_DAYS = "days";
 
 struct ArchiveConf {
-    using CriteriaT = std::map<std::string, int>;
+    using CriteriaT = std::map<std::string, int64_t>;
 
     explicit ArchiveConf(const std::string& type, const std::string& criterias = std::string());
 

core/src/db/Types.h

@@ -27,8 +27,7 @@
 namespace milvus {
 namespace engine {
 
-using IDNumber = faiss::Index::idx_t;
-
+typedef int64_t IDNumber;
 typedef IDNumber* IDNumberPtr;
 typedef std::vector<IDNumber> IDNumbers;
 

core/src/db/engine/ExecutionEngine.h

@@ -26,6 +26,7 @@
 namespace milvus {
 namespace engine {
 
+// TODO(linxj): replace with VecIndex::IndexType
 enum class EngineType {
     INVALID = 0,
     FAISS_IDMAP = 1,
@@ -33,7 +34,10 @@ enum class EngineType {
     FAISS_IVFSQ8,
     NSG_MIX,
     FAISS_IVFSQ8H,
-    MAX_VALUE = FAISS_IVFSQ8H,
+    FAISS_PQ,
+    SPTAG_KDT,
+    SPTAG_BKT,
+    MAX_VALUE = SPTAG_BKT,
 };
 
 enum class MetricType {

core/src/db/engine/ExecutionEngineImpl.cpp

@@ -25,6 +25,7 @@
 #include "utils/CommonUtil.h"
 #include "utils/Exception.h"
 #include "utils/Log.h"
+
 #include "wrapper/ConfAdapter.h"
 #include "wrapper/ConfAdapterMgr.h"
 #include "wrapper/VecImpl.h"
@@ -92,11 +93,19 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
             break;
         }
         case EngineType::FAISS_IVFFLAT: {
+#ifdef MILVUS_CPU_VERSION
+            index = GetVecIndexFactory(IndexType::FAISS_IVFFLAT_CPU);
+#else
             index = GetVecIndexFactory(IndexType::FAISS_IVFFLAT_MIX);
+#endif
             break;
         }
         case EngineType::FAISS_IVFSQ8: {
+#ifdef MILVUS_CPU_VERSION
+            index = GetVecIndexFactory(IndexType::FAISS_IVFSQ8_CPU);
+#else
             index = GetVecIndexFactory(IndexType::FAISS_IVFSQ8_MIX);
+#endif
             break;
         }
         case EngineType::NSG_MIX: {
@@ -107,6 +116,22 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
             index = GetVecIndexFactory(IndexType::FAISS_IVFSQ8_HYBRID);
             break;
         }
+        case EngineType::FAISS_PQ: {
+#ifdef MILVUS_CPU_VERSION
+            index = GetVecIndexFactory(IndexType::FAISS_IVFPQ_CPU);
+#else
+            index = GetVecIndexFactory(IndexType::FAISS_IVFPQ_MIX);
+#endif
+            break;
+        }
+        case EngineType::SPTAG_KDT: {
+            index = GetVecIndexFactory(IndexType::SPTAG_KDT_RNT_CPU);
+            break;
+        }
+        case EngineType::SPTAG_BKT: {
+            index = GetVecIndexFactory(IndexType::SPTAG_BKT_RNT_CPU);
+            break;
+        }
         default: {
             ENGINE_LOG_ERROR << "Unsupported index type";
             return nullptr;
@@ -127,7 +152,14 @@ ExecutionEngineImpl::HybridLoad() const {
     }
 
     const std::string key = location_ + ".quantizer";
-    std::vector<uint64_t> gpus = scheduler::get_gpu_pool();
+
+    server::Config& config = server::Config::GetInstance();
+    std::vector<int64_t> gpus;
+    Status s = config.GetGpuResourceConfigSearchResources(gpus);
+    if (!s.ok()) {
+        ENGINE_LOG_ERROR << s.message();
+        return;
+    }
 
     // cache hit
     {
@@ -309,18 +341,36 @@ ExecutionEngineImpl::CopyToGpu(uint64_t device_id, bool hybrid) {
         return Status::OK();
     }
 #endif
-    try {
-        index_ = index_->CopyToGpu(device_id);
-        ENGINE_LOG_DEBUG << "CPU to GPU" << device_id;
-    } catch (std::exception& e) {
-        ENGINE_LOG_ERROR << e.what();
-        return Status(DB_ERROR, e.what());
+
+    auto index = std::static_pointer_cast<VecIndex>(cache::GpuCacheMgr::GetInstance(device_id)->GetIndex(location_));
+    bool already_in_cache = (index != nullptr);
+    if (already_in_cache) {
+        index_ = index;
+    } else {
+        if (index_ == nullptr) {
+            ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to copy to gpu";
+            return Status(DB_ERROR, "index is null");
+        }
+
+        try {
+            index_ = index_->CopyToGpu(device_id);
+            ENGINE_LOG_DEBUG << "CPU to GPU" << device_id;
+        } catch (std::exception& e) {
+            ENGINE_LOG_ERROR << e.what();
+            return Status(DB_ERROR, e.what());
+        }
+    }
+
+    if (!already_in_cache) {
+        GpuCache(device_id);
     }
+
     return Status::OK();
 }
 
 Status
 ExecutionEngineImpl::CopyToIndexFileToGpu(uint64_t device_id) {
+    gpu_num_ = device_id;
     auto to_index_data = std::make_shared<ToIndexData>(PhysicalSize());
     cache::DataObjPtr obj = std::static_pointer_cast<cache::DataObj>(to_index_data);
     milvus::cache::GpuCacheMgr::GetInstance(device_id)->InsertItem(location_, obj);
@@ -544,12 +594,16 @@ ExecutionEngineImpl::GpuCache(uint64_t gpu_id) {
 Status
 ExecutionEngineImpl::Init() {
     server::Config& config = server::Config::GetInstance();
-    Status s = config.GetResourceConfigIndexBuildDevice(gpu_num_);
-    if (!s.ok()) {
-        return s;
+    std::vector<int64_t> gpu_ids;
+    Status s = config.GetGpuResourceConfigBuildIndexResources(gpu_ids);
+    for (auto id : gpu_ids) {
+        if (gpu_num_ == id) {
+            return Status::OK();
+        }
     }
 
-    return Status::OK();
+    std::string msg = "Invalid gpu_num";
+    return Status(SERVER_INVALID_ARGUMENT, msg);
 }
 
 }  // namespace engine

core/src/db/engine/ExecutionEngineImpl.h

@@ -122,8 +122,8 @@ class ExecutionEngineImpl : public ExecutionEngine {
     int64_t dim_;
     std::string location_;
 
-    int32_t nlist_ = 0;
-    int32_t gpu_num_ = 0;
+    int64_t nlist_ = 0;
+    int64_t gpu_num_ = 0;
 };
 
 }  // namespace engine

core/src/db/meta/Meta.h

@@ -50,14 +50,11 @@ class Meta {
     virtual Status
     AllTables(std::vector<TableSchema>& table_schema_array) = 0;
 
-    virtual Status
-    UpdateTableIndex(const std::string& table_id, const TableIndex& index) = 0;
-
     virtual Status
     UpdateTableFlag(const std::string& table_id, int64_t flag) = 0;
 
     virtual Status
-    DeleteTable(const std::string& table_id) = 0;
+    DropTable(const std::string& table_id) = 0;
 
     virtual Status
     DeleteTableFiles(const std::string& table_id) = 0;
@@ -66,32 +63,47 @@ class Meta {
     CreateTableFile(TableFileSchema& file_schema) = 0;
 
     virtual Status
-    DropPartitionsByDates(const std::string& table_id, const DatesT& dates) = 0;
+    DropDataByDate(const std::string& table_id, const DatesT& dates) = 0;
 
     virtual Status
     GetTableFiles(const std::string& table_id, const std::vector<size_t>& ids, TableFilesSchema& table_files) = 0;
 
+    virtual Status
+    UpdateTableFile(TableFileSchema& file_schema) = 0;
+
+    virtual Status
+    UpdateTableFiles(TableFilesSchema& files) = 0;
+
+    virtual Status
+    UpdateTableIndex(const std::string& table_id, const TableIndex& index) = 0;
+
     virtual Status
     UpdateTableFilesToIndex(const std::string& table_id) = 0;
 
     virtual Status
-    UpdateTableFile(TableFileSchema& file_schema) = 0;
+    DescribeTableIndex(const std::string& table_id, TableIndex& index) = 0;
 
     virtual Status
-    UpdateTableFiles(TableFilesSchema& files) = 0;
+    DropTableIndex(const std::string& table_id) = 0;
 
     virtual Status
-    FilesToSearch(const std::string& table_id, const std::vector<size_t>& ids, const DatesT& dates,
-                  DatePartionedTableFilesSchema& files) = 0;
+    CreatePartition(const std::string& table_name, const std::string& partition_name, const std::string& tag) = 0;
 
     virtual Status
-    FilesToMerge(const std::string& table_id, DatePartionedTableFilesSchema& files) = 0;
+    DropPartition(const std::string& partition_name) = 0;
 
     virtual Status
-    Size(uint64_t& result) = 0;
+    ShowPartitions(const std::string& table_name, std::vector<meta::TableSchema>& partiton_schema_array) = 0;
 
     virtual Status
-    Archive() = 0;
+    GetPartitionName(const std::string& table_name, const std::string& tag, std::string& partition_name) = 0;
+
+    virtual Status
+    FilesToSearch(const std::string& table_id, const std::vector<size_t>& ids, const DatesT& dates,
+                  DatePartionedTableFilesSchema& files) = 0;
+
+    virtual Status
+    FilesToMerge(const std::string& table_id, DatePartionedTableFilesSchema& files) = 0;
 
     virtual Status
     FilesToIndex(TableFilesSchema&) = 0;
@@ -101,10 +113,10 @@ class Meta {
                 std::vector<std::string>& file_ids) = 0;
 
     virtual Status
-    DescribeTableIndex(const std::string& table_id, TableIndex& index) = 0;
+    Size(uint64_t& result) = 0;
 
     virtual Status
-    DropTableIndex(const std::string& table_id) = 0;
+    Archive() = 0;
 
     virtual Status
     CleanUp() = 0;

core/src/db/meta/MetaTypes.h

@@ -19,6 +19,7 @@
 
 #include "db/Constants.h"
 #include "db/engine/ExecutionEngine.h"
+#include "src/version.h"
 
 #include <map>
 #include <memory>
@@ -33,6 +34,7 @@ constexpr int32_t DEFAULT_ENGINE_TYPE = (int)EngineType::FAISS_IDMAP;
 constexpr int32_t DEFAULT_NLIST = 16384;
 constexpr int32_t DEFAULT_METRIC_TYPE = (int)MetricType::L2;
 constexpr int32_t DEFAULT_INDEX_FILE_SIZE = ONE_GB;
+constexpr char CURRENT_VERSION[] = MILVUS_VERSION;
 
 constexpr int64_t FLAG_MASK_NO_USERID = 0x1;
 constexpr int64_t FLAG_MASK_HAS_USERID = 0x1 << 1;
@@ -57,6 +59,9 @@ struct TableSchema {
     int32_t engine_type_ = DEFAULT_ENGINE_TYPE;
     int32_t nlist_ = DEFAULT_NLIST;
     int32_t metric_type_ = DEFAULT_METRIC_TYPE;
+    std::string owner_table_;
+    std::string partition_tag_;
+    std::string version_ = CURRENT_VERSION;
 };  // TableSchema
 
 struct TableFileSchema {

core/src/db/meta/MySQLMetaImpl.h

@@ -49,7 +49,7 @@ class MySQLMetaImpl : public Meta {
     AllTables(std::vector<TableSchema>& table_schema_array) override;
 
     Status
-    DeleteTable(const std::string& table_id) override;
+    DropTable(const std::string& table_id) override;
 
     Status
     DeleteTableFiles(const std::string& table_id) override;
@@ -58,21 +58,26 @@ class MySQLMetaImpl : public Meta {
     CreateTableFile(TableFileSchema& file_schema) override;
 
     Status
-    DropPartitionsByDates(const std::string& table_id, const DatesT& dates) override;
+    DropDataByDate(const std::string& table_id, const DatesT& dates) override;
 
     Status
     GetTableFiles(const std::string& table_id, const std::vector<size_t>& ids, TableFilesSchema& table_files) override;
 
-    Status
-    FilesByType(const std::string& table_id, const std::vector<int>& file_types,
-                std::vector<std::string>& file_ids) override;
-
     Status
     UpdateTableIndex(const std::string& table_id, const TableIndex& index) override;
 
     Status
     UpdateTableFlag(const std::string& table_id, int64_t flag) override;
 
+    Status
+    UpdateTableFile(TableFileSchema& file_schema) override;
+
+    Status
+    UpdateTableFilesToIndex(const std::string& table_id) override;
+
+    Status
+    UpdateTableFiles(TableFilesSchema& files) override;
+
     Status
     DescribeTableIndex(const std::string& table_id, TableIndex& index) override;
 
@@ -80,13 +85,16 @@ class MySQLMetaImpl : public Meta {
     DropTableIndex(const std::string& table_id) override;
 
     Status
-    UpdateTableFile(TableFileSchema& file_schema) override;
+    CreatePartition(const std::string& table_id, const std::string& partition_name, const std::string& tag) override;
 
     Status
-    UpdateTableFilesToIndex(const std::string& table_id) override;
+    DropPartition(const std::string& partition_name) override;
 
     Status
-    UpdateTableFiles(TableFilesSchema& files) override;
+    ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partiton_schema_array) override;
+
+    Status
+    GetPartitionName(const std::string& table_id, const std::string& tag, std::string& partition_name) override;
 
     Status
     FilesToSearch(const std::string& table_id, const std::vector<size_t>& ids, const DatesT& dates,
@@ -98,6 +106,10 @@ class MySQLMetaImpl : public Meta {
     Status
     FilesToIndex(TableFilesSchema&) override;
 
+    Status
+    FilesByType(const std::string& table_id, const std::vector<int>& file_types,
+                std::vector<std::string>& file_ids) override;
+
     Status
     Archive() override;
 

core/src/db/meta/SqliteMetaImpl.h

@@ -49,7 +49,7 @@ class SqliteMetaImpl : public Meta {
     AllTables(std::vector<TableSchema>& table_schema_array) override;
 
     Status
-    DeleteTable(const std::string& table_id) override;
+    DropTable(const std::string& table_id) override;
 
     Status
     DeleteTableFiles(const std::string& table_id) override;
@@ -58,21 +58,26 @@ class SqliteMetaImpl : public Meta {
     CreateTableFile(TableFileSchema& file_schema) override;
 
     Status
-    DropPartitionsByDates(const std::string& table_id, const DatesT& dates) override;
+    DropDataByDate(const std::string& table_id, const DatesT& dates) override;
 
     Status
     GetTableFiles(const std::string& table_id, const std::vector<size_t>& ids, TableFilesSchema& table_files) override;
 
-    Status
-    FilesByType(const std::string& table_id, const std::vector<int>& file_types,
-                std::vector<std::string>& file_ids) override;
-
     Status
     UpdateTableIndex(const std::string& table_id, const TableIndex& index) override;
 
     Status
     UpdateTableFlag(const std::string& table_id, int64_t flag) override;
 
+    Status
+    UpdateTableFile(TableFileSchema& file_schema) override;
+
+    Status
+    UpdateTableFilesToIndex(const std::string& table_id) override;
+
+    Status
+    UpdateTableFiles(TableFilesSchema& files) override;
+
     Status
     DescribeTableIndex(const std::string& table_id, TableIndex& index) override;
 
@@ -80,13 +85,16 @@ class SqliteMetaImpl : public Meta {
     DropTableIndex(const std::string& table_id) override;
 
     Status
-    UpdateTableFilesToIndex(const std::string& table_id) override;
+    CreatePartition(const std::string& table_id, const std::string& partition_name, const std::string& tag) override;
 
     Status
-    UpdateTableFile(TableFileSchema& file_schema) override;
+    DropPartition(const std::string& partition_name) override;
 
     Status
-    UpdateTableFiles(TableFilesSchema& files) override;
+    ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partiton_schema_array) override;
+
+    Status
+    GetPartitionName(const std::string& table_id, const std::string& tag, std::string& partition_name) override;
 
     Status
     FilesToSearch(const std::string& table_id, const std::vector<size_t>& ids, const DatesT& dates,
@@ -99,11 +107,15 @@ class SqliteMetaImpl : public Meta {
     FilesToIndex(TableFilesSchema&) override;
 
     Status
-    Archive() override;
+    FilesByType(const std::string& table_id, const std::vector<int>& file_types,
+                std::vector<std::string>& file_ids) override;
 
     Status
     Size(uint64_t& result) override;
 
+    Status
+    Archive() override;
+
     Status
     CleanUp() override;
 

core/src/grpc/milvus.proto

@@ -5,14 +5,21 @@ import "status.proto";
 package milvus.grpc;
 
 /**
- * @brief Table Name
+ * @brief Table name
  */
 message TableName {
     string table_name = 1;
 }
 
 /**
- * @brief Table Name List
+ * @brief Partition name
+ */
+message PartitionName {
+    string partition_name = 1;
+}
+
+/**
+ * @brief Table name list
  */
 message TableNameList {
     Status status = 1;
@@ -20,7 +27,7 @@ message TableNameList {
 }
 
 /**
- * @brief Table Schema
+ * @brief Table schema
  */
 message TableSchema {
     Status status = 1;
@@ -31,7 +38,24 @@ message TableSchema {
 }
 
 /**
- * @brief Range Schema
+ * @brief Params of partition
+ */
+message PartitionParam {
+    string table_name = 1;
+    string partition_name = 2;
+    string tag = 3;
+}
+
+/**
+ * @brief Partition list
+ */
+message PartitionList {
+    Status status = 1;
+    repeated PartitionParam partition_array = 2;
+}
+
+/**
+ * @brief Range schema
  */
 message Range {
     string start_value = 1;
@@ -46,12 +70,13 @@ message RowRecord {
 }
 
 /**
- * @brief params to be inserted
+ * @brief Params to be inserted
  */
 message InsertParam {
     string table_name = 1;
     repeated RowRecord row_record_array = 2;
     repeated int64 row_id_array = 3;            //optional
+    string partition_tag = 4;
 }
 
 /**
@@ -63,7 +88,7 @@ message VectorIds {
 }
 
 /**
- * @brief params for searching vector
+ * @brief Params for searching vector
  */
 message SearchParam {
     string table_name = 1;
@@ -71,10 +96,11 @@ message SearchParam {
     repeated Range query_range_array = 3;
     int64 topk = 4;
     int64 nprobe = 5;
+    repeated string partition_tag_array = 6;
 }
 
 /**
- * @brief params for searching vector in files
+ * @brief Params for searching vector in files
  */
 message SearchInFilesParam {
     repeated string file_id_array = 1;
@@ -92,7 +118,7 @@ message TopKQueryResult {
 }
 
 /**
- * @brief Server String Reply
+ * @brief Server string Reply
  */
 message StringReply {
     Status status = 1;
@@ -116,7 +142,7 @@ message TableRowCount {
 }
 
 /**
- * @brief Give Server Command
+ * @brief Give server Command
  */
 message Command {
     string cmd = 1;
@@ -142,169 +168,173 @@ message IndexParam {
 }
 
 /**
- * @brief table name and range for DeleteByRange
+ * @brief table name and range for DeleteByDate
  */
-message DeleteByRangeParam {
+message DeleteByDateParam {
     Range range = 1;
     string table_name = 2;
 }
 
 service MilvusService {
     /**
-     * @brief Create table method
-     *
-     * This method is used to create table
+     * @brief This method is used to create table
      *
-     * @param param, use to provide table information to be created.
+     * @param TableSchema, use to provide table information to be created.
      *
+     * @return Status
      */
     rpc CreateTable(TableSchema) returns (Status){}
 
     /**
-     * @brief Test table existence method
+     * @brief This method is used to test table existence.
      *
-     * This method is used to test table existence.
-     *
-     * @param table_name, table name is going to be tested.
+     * @param TableName, table name is going to be tested.
      *
+     * @return BoolReply
      */
     rpc HasTable(TableName) returns (BoolReply) {}
 
     /**
-     * @brief Delete table method
-     *
-     * This method is used to delete table.
+     * @brief This method is used to get table schema.
      *
-     * @param table_name, table name is going to be deleted.
+     * @param TableName, target table name.
      *
+     * @return TableSchema
      */
-    rpc DropTable(TableName) returns (Status) {}
+    rpc DescribeTable(TableName) returns (TableSchema) {}
 
     /**
-     * @brief Build index by table method
+     * @brief This method is used to get table schema.
      *
-     * This method is used to build index by table in sync mode.
-     *
-     * @param table_name, table is going to be built index.
+     * @param TableName, target table name.
      *
+     * @return TableRowCount
      */
-    rpc CreateIndex(IndexParam) returns (Status) {}
+    rpc CountTable(TableName) returns (TableRowCount) {}
 
     /**
-     * @brief Add vector array to table
-     *
-     * This method is used to add vector array to table.
+     * @brief This method is used to list all tables.
      *
-     * @param table_name, table_name is inserted.
-     * @param record_array, vector array is inserted.
+     * @param Command, dummy parameter.
      *
-     * @return vector id array
+     * @return TableNameList
      */
-    rpc Insert(InsertParam) returns (VectorIds) {}
+    rpc ShowTables(Command) returns (TableNameList) {}
 
     /**
-     * @brief Query vector
+     * @brief This method is used to delete table.
      *
-     * This method is used to query vector in table.
+     * @param TableName, table name is going to be deleted.
      *
-     * @param table_name, table_name is queried.
-     * @param query_record_array, all vector are going to be queried.
-     * @param query_range_array, optional ranges for conditional search. If not specified, search whole table
-     * @param topk, how many similarity vectors will be searched.
+     * @return TableNameList
+     */
+    rpc DropTable(TableName) returns (Status) {}
+
+    /**
+     * @brief This method is used to build index by table in sync mode.
      *
-     * @return query result array.
+     * @param IndexParam, index paramters.
+     *
+     * @return Status
      */
-    rpc Search(SearchParam) returns (TopKQueryResult) {}
+    rpc CreateIndex(IndexParam) returns (Status) {}
 
     /**
-     * @brief Internal use query interface
+     * @brief This method is used to describe index
+     *
+     * @param TableName, target table name.
      *
-     * This method is used to query vector in specified files.
+     * @return IndexParam
+     */
+    rpc DescribeIndex(TableName) returns (IndexParam) {}
+
+    /**
+     * @brief This method is used to drop index
      *
-     * @param file_id_array, specified files id array, queried.
-     * @param query_record_array, all vector are going to be queried.
-     * @param query_range_array, optional ranges for conditional search. If not specified, search whole table
-     * @param topk, how many similarity vectors will be searched.
+     * @param TableName, target table name.
      *
-     * @return query result array.
+     * @return Status
      */
-    rpc SearchInFiles(SearchInFilesParam) returns (TopKQueryResult) {}
+    rpc DropIndex(TableName) returns (Status) {}
 
     /**
-     * @brief Get table schema
+     * @brief This method is used to create partition
      *
-     * This method is used to get table schema.
+     * @param PartitionParam, partition parameters.
      *
-     * @param table_name, target table name.
+     * @return Status
+     */
+    rpc CreatePartition(PartitionParam) returns (Status) {}
+
+    /**
+     * @brief This method is used to show partition information
      *
-     * @return table schema
+     * @param TableName, target table name.
+     *
+     * @return PartitionList
      */
-    rpc DescribeTable(TableName) returns (TableSchema) {}
+    rpc ShowPartitions(TableName) returns (PartitionList) {}
 
     /**
-     * @brief Get table schema
+     * @brief This method is used to drop partition
      *
-     * This method is used to get table schema.
+     * @param PartitionParam, target partition.
      *
-     * @param table_name, target table name.
+     * @return Status
+     */
+    rpc DropPartition(PartitionParam) returns (Status) {}
+
+    /**
+     * @brief This method is used to add vector array to table.
+     *
+     * @param InsertParam, insert parameters.
      *
-     * @return table schema
+     * @return VectorIds
      */
-    rpc CountTable(TableName) returns (TableRowCount) {}
+    rpc Insert(InsertParam) returns (VectorIds) {}
 
     /**
-     * @brief List all tables in database
+     * @brief This method is used to query vector in table.
      *
-     * This method is used to list all tables.
+     * @param SearchParam, search parameters.
      *
+     * @return TopKQueryResult
+     */
+    rpc Search(SearchParam) returns (TopKQueryResult) {}
+
+    /**
+     * @brief This method is used to query vector in specified files.
      *
-     * @return table names.
+     * @param SearchInFilesParam, search in files paremeters.
+     *
+     * @return TopKQueryResult
      */
-    rpc ShowTables(Command) returns (TableNameList) {}
+    rpc SearchInFiles(SearchInFilesParam) returns (TopKQueryResult) {}
 
     /**
-     * @brief Give the server status
+     * @brief This method is used to give the server status.
      *
-     * This method is used to give the server status.
+     * @param Command, command string
      *
-     * @return Server status.
+     * @return StringReply
      */
     rpc Cmd(Command) returns (StringReply) {}
 
      /**
-     * @brief delete table by range
+     * @brief This method is used to delete vector by date range
      *
-     * This method is used to delete vector by range
+     * @param DeleteByDateParam, delete parameters.
      *
-     * @return rpc status.
+     * @return status
      */
-     rpc DeleteByRange(DeleteByRangeParam) returns (Status) {}
+     rpc DeleteByDate(DeleteByDateParam) returns (Status) {}
 
      /**
-      * @brief preload table
+      * @brief This method is used to preload table
       *
-      * This method is used to preload table
+      * @param TableName, target table name.
       *
-      * @return Status.
+      * @return Status
       */
      rpc PreloadTable(TableName) returns (Status) {}
-
-     /**
-      * @brief describe index
-      *
-      * This method is used to describe index
-      *
-      * @return Status.
-      */
-     rpc DescribeIndex(TableName) returns (IndexParam) {}
-
-     /**
-      * @brief drop index
-      *
-      * This method is used to drop index
-      *
-      * @return Status.
-      */
-     rpc DropIndex(TableName) returns (Status) {}
-
 }

core/src/index/build.sh

@@ -5,7 +5,7 @@ BUILD_UNITTEST="OFF"
 INSTALL_PREFIX=$(pwd)/cmake_build
 MAKE_CLEAN="OFF"
 PROFILING="OFF"
-BUILD_FAISS_WITH_MKL="OFF"
+FAISS_WITH_MKL="OFF"
 USE_JFROG_CACHE="OFF"
 
 while getopts "p:d:t:uhrcgmj" arg
@@ -31,7 +31,7 @@ do
                 PROFILING="ON"
                 ;;
              m)
-                BUILD_FAISS_WITH_MKL="ON"
+                FAISS_WITH_MKL="ON"
                 ;;
              j)
                 USE_JFROG_CACHE="ON"
@@ -74,7 +74,7 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then
     -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
     -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
     -DMILVUS_ENABLE_PROFILING=${PROFILING} \
-    -DBUILD_FAISS_WITH_MKL=${BUILD_FAISS_WITH_MKL} \
+    -DFAISS_WITH_MKL=${FAISS_WITH_MKL} \
     -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
     ../"
     echo ${CMAKE_CMD}

core/src/index/knowhere/knowhere/common/Log.h

@@ -17,7 +17,7 @@
 
 #pragma once
 
-#include "external/easyloggingpp/easylogging++.h"
+#include "easyloggingpp/easylogging++.h"
 
 namespace knowhere {