solve conflicts

Former-commit-id: 79d4e00b90019c196b065b2654535c689dec3900

ci/jenkinsfile/cleanup_staging.groovy

+try {
+    def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true
+    if (!result) {
+        sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}"
+    }
+} catch (exc) {
+    def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true
+    if (!result) {
+        sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}"
+    }
+    throw exc
+}
+

ci/jenkinsfile/deploy2staging.groovy

+try {
+    sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
+    sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus'
+    sh 'helm repo update'
+    dir ("milvus-helm") {
+        checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
+        dir ("milvus/milvus-gpu") {
+            sh "helm install --wait --timeout 300 --set engine.image.repository="zilliz.azurecr.cn/milvus/engine" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.4.0 ."
+        }
+    }
+} catch (exc) {
+    echo 'Helm running failed!'
+    sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}"
+    throw exc
+}
+

ci/jenkinsfile/dev_test.groovy

@@ -3,7 +3,7 @@ timeout(time: 40, unit: 'MINUTES') {
         dir ("${PROJECT_NAME}_test") {
             checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
             sh 'python3 -m pip install -r requirements.txt'
-            sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local"
+            sh "pytest . --alluredir="test_out/dev/single/sqlite" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local"
         }
 
         // mysql database backend test
@@ -16,11 +16,11 @@ timeout(time: 40, unit: 'MINUTES') {
         }
         dir ("milvus-helm") {
             dir ("milvus/milvus-gpu") {
-                sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.3.1 ."
+                sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.4.0 ."
             }
         }
         dir ("${PROJECT_NAME}_test") {
-            sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local"
+            sh "pytest . --alluredir="test_out/dev/single/mysql" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local"
         }
     } catch (exc) {
         echo 'Milvus Test Failed !'

ci/jenkinsfile/staging_test.groovy

+timeout(time: 40, unit: 'MINUTES') {
+    try {
+        dir ("${PROJECT_NAME}_test") {
+            checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
+            sh 'python3 -m pip install -r requirements.txt'
+            def service_ip = sh (script: "kubectl get svc --namespace milvus-1 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine --template \"{{range .status.loadBalancer.ingress}}{{.ip}}{{end}}\"",returnStdout: true).trim()
+            sh "pytest . --alluredir="test_out/staging/single/sqlite" --ip ${service_ip}"
+        }
+
+        // mysql database backend test
+        load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy"
+
+        if (!fileExists('milvus-helm')) {
+            dir ("milvus-helm") {
+                checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
+            }
+        }
+        dir ("milvus-helm") {
+            dir ("milvus/milvus-gpu") {
+                sh "helm install --wait --timeout 300 --set engine.image.repository="zilliz.azurecr.cn/milvus/engine" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.4.0 ."
+            }
+        }
+        dir ("${PROJECT_NAME}_test") {
+            def service_ip = sh (script: "kubectl get svc --namespace milvus-2 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine --template \"{{range .status.loadBalancer.ingress}}{{.ip}}{{end}}\"",returnStdout: true).trim()
+            sh "pytest . --alluredir="test_out/staging/single/mysql" --ip ${service_ip}"
+        }
+    } catch (exc) {
+        echo 'Milvus Test Failed !'
+        throw exc
+    }
+}

ci/jenkinsfile/upload_dev_test_out.groovy

 timeout(time: 5, unit: 'MINUTES') {
     dir ("${PROJECT_NAME}_test") {
-        if (fileExists('test_out')) {
+        if (fileExists('test_out/dev')) {
             def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy"
-            fileTransfer.FileTransfer("test_out/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage')
+            fileTransfer.FileTransfer("test_out/dev/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage')
             if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
                 echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\""
             }

ci/jenkinsfile/upload_staging_test_out.groovy

+timeout(time: 5, unit: 'MINUTES') {
+    dir ("${PROJECT_NAME}_test") {
+        if (fileExists('test_out/staging')) {
+            def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy"
+            fileTransfer.FileTransfer("test_out/staging/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage')
+            if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
+                echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\""
+            }
+        } else {
+            error("Milvus Dev Test Out directory don't exists!")
+        }
+    }
+}

ci/nightly_main_jenkinsfile

@@ -208,17 +208,17 @@ spec:
                                 }
                                 success {
                                     script {
-                                        echo "Milvus Single Node CI/CD success !"
+                                        echo "Milvus Deploy to Dev Single Node CI/CD success !"
                                     }
                                 }
                                 aborted {
                                     script {
-                                        echo "Milvus Single Node CI/CD aborted !"
+                                        echo "Milvus Deploy to Dev Single Node CI/CD aborted !"
                                     }
                                 }
                                 failure {
                                     script {
-                                        echo "Milvus Single Node CI/CD failure !"
+                                        echo "Milvus Deploy to Dev Single Node CI/CD failure !"
                                     }
                                 }
                             }
@@ -300,17 +300,114 @@ spec:
                                 }
                                 success {
                                     script {
-                                        echo "Milvus Cluster CI/CD success !"
+                                        echo "Milvus Deploy to Dev Cluster CI/CD success !"
                                     }
                                 }
                                 aborted {
                                     script {
-                                        echo "Milvus Cluster CI/CD aborted !"
+                                        echo "Milvus Deploy to Dev Cluster CI/CD aborted !"
                                     }
                                 }
                                 failure {
                                     script {
-                                        echo "Milvus Cluster CI/CD failure !"
+                                        echo "Milvus Deploy to Dev Cluster CI/CD failure !"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+
+                stage("Deploy to Staging") {
+                    parallel {
+                        stage("Single Node") {
+                            agent {
+                                kubernetes {
+                                    label 'dev-test'
+                                    defaultContainer 'jnlp'
+                                    yaml """
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: milvus
+    componet: test
+spec:
+  containers:
+  - name: milvus-testframework
+    image: registry.zilliz.com/milvus/milvus-test:v0.2
+    command:
+    - cat
+    tty: true
+    volumeMounts:
+    - name: kubeconf
+      mountPath: /root/.kube/
+      readOnly: true
+  volumes:
+  - name: kubeconf
+    secret:
+      secretName: aks-gpu-cluster-config
+"""
+                                }
+                            }
+
+                            stages {
+                                stage("Deploy to Staging") {
+                                    steps {
+                                        gitlabCommitStatus(name: 'Deloy to Staging') {
+                                            container('milvus-testframework') {
+                                                script {
+                                                    load "${env.WORKSPACE}/ci/jenkinsfile/deploy2staging.groovy"
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
+                                stage("Staging Test") {
+                                    steps {
+                                        gitlabCommitStatus(name: 'Staging Test') {
+                                            container('milvus-testframework') {
+                                                script {
+                                                    load "${env.WORKSPACE}/ci/jenkinsfile/staging_test.groovy"
+                                                    load "${env.WORKSPACE}/ci/jenkinsfile/upload_staging_test_out.groovy"
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
+                                stage ("Cleanup Staging") {
+                                    steps {
+                                        gitlabCommitStatus(name: 'Cleanup Staging') {
+                                            container('milvus-testframework') {
+                                                script {
+                                                    load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy"
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                            post {
+                                always {
+                                    container('milvus-testframework') {
+                                        script {
+                                            load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy"
+                                        }
+                                    }
+                                }
+                                success {
+                                    script {
+                                        echo "Milvus Deploy to Staging Single Node CI/CD success !"
+                                    }
+                                }
+                                aborted {
+                                    script {
+                                        echo "Milvus Deploy to Staging Single Node CI/CD aborted !"
+                                    }
+                                }
+                                failure {
+                                    script {
+                                        echo "Milvus Deploy to Staging Single Node CI/CD failure !"
                                     }
                                 }
                             }

cpp/CHANGELOG.md

@@ -56,6 +56,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-414 - Add TaskType in Scheduler::Task
 - MS-415 - Add command tasktable to dump all tasktables
 - MS-418 - Update server_config.template file, set CPU compute only default
+- MS-419 - Move index_file_size from IndexParam to TableSchema
 
 ## New Feature
 - MS-343 - Implement ResourceMgr

cpp/src/db/DBImpl.cpp

@@ -55,7 +55,9 @@ DBImpl::DBImpl(const Options& options)
 }
 
 Status DBImpl::CreateTable(meta::TableSchema& table_schema) {
-    return meta_ptr_->CreateTable(table_schema);
+    meta::TableSchema temp_schema = table_schema;
+    temp_schema.index_file_size_ *= ONE_MB;
+    return meta_ptr_->CreateTable(temp_schema);
 }
 
 Status DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& dates) {

cpp/src/db/Types.h

@@ -24,7 +24,6 @@ typedef std::vector<QueryResult> QueryResults;
 struct TableIndex {
     int32_t engine_type_ = (int)EngineType::FAISS_IDMAP;
     int32_t nlist_ = 16384;
-    int32_t index_file_size_ = 1024; //MB
     int32_t metric_type_ = (int)MetricType::L2;
 };
 

cpp/src/db/Utils.cpp

@@ -149,7 +149,6 @@ Status DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema&
 bool IsSameIndex(const TableIndex& index1, const TableIndex& index2) {
     return index1.engine_type_ == index2.engine_type_
         && index1.nlist_ == index2.nlist_
-        && index1.index_file_size_ == index2.index_file_size_
         && index1.metric_type_ == index2.metric_type_;
 }
 

cpp/src/db/meta/MetaTypes.h

@@ -40,9 +40,9 @@ struct TableSchema {
     uint16_t dimension_ = 0;
     int64_t created_on_ = 0;
     int64_t flag_ = 0;
+    int32_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE;
     int32_t engine_type_ = DEFAULT_ENGINE_TYPE;
     int32_t nlist_ = DEFAULT_NLIST;
-    int32_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE;
     int32_t metric_type_ = DEFAULT_METRIC_TYPE;
 }; // TableSchema
 
@@ -69,9 +69,9 @@ struct TableFileSchema {
     std::string location_;
     int64_t updated_time_ = 0;
     int64_t created_on_ = 0;
+    int64_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE; //not persist to meta
     int32_t engine_type_ = DEFAULT_ENGINE_TYPE;
     int32_t nlist_ = DEFAULT_NLIST; //not persist to meta
-    int32_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE; //not persist to meta
     int32_t metric_type_ = DEFAULT_METRIC_TYPE; //not persist to meta
 }; // TableFileSchema
 

cpp/src/db/meta/MySQLMetaImpl.cpp

@@ -138,9 +138,9 @@ Status MySQLMetaImpl::Initialize() {
                                 "dimension SMALLINT NOT NULL, " <<
                                 "created_on BIGINT NOT NULL, " <<
                                 "flag BIGINT DEFAULT 0 NOT NULL, " <<
+                                "index_file_size INT DEFAULT 1024 NOT NULL, " <<
                                 "engine_type INT DEFAULT 1 NOT NULL, " <<
                                 "nlist INT DEFAULT 16384 NOT NULL, " <<
-                                "index_file_size INT DEFAULT 1024 NOT NULL, " <<
                                 "metric_type INT DEFAULT 1 NOT NULL);";
 
                 ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str();
@@ -407,7 +407,6 @@ Status MySQLMetaImpl::UpdateTableIndexParam(const std::string &table_id, const T
                                            "created_on = " << created_on << ", " <<
                                            "engine_type_ = " << index.engine_type_ << ", " <<
                                            "nlist = " << index.nlist_ << ", " <<
-                                           "index_file_size = " << index.index_file_size_*ONE_MB << ", " <<
                                            "metric_type = " << index.metric_type_ << " " <<
                                            "WHERE id = " << quote << table_id << ";";
 
@@ -504,7 +503,6 @@ Status MySQLMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex
 
                 index.engine_type_ = resRow["engine_type"];
                 index.nlist_ = resRow["nlist"];
-                index.index_file_size_ = resRow["index_file_size"]/ONE_MB;
                 index.metric_type_ = resRow["metric_type"];
             } else {
                 return Status::NotFound("Table " + table_id + " not found");
@@ -694,12 +692,12 @@ Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) {
 
             table_schema.dimension_ = resRow["dimension"];
 
+            table_schema.index_file_size_ = resRow["index_file_size"];
+
             table_schema.engine_type_ = resRow["engine_type"];
 
             table_schema.nlist_ = resRow["nlist"];
 
-            table_schema.index_file_size_ = resRow["index_file_size"];
-
             table_schema.metric_type_ = resRow["metric_type"];
         } else {
             return Status::NotFound("Table " + table_schema.table_id_ + " not found");
@@ -790,12 +788,12 @@ Status MySQLMetaImpl::AllTables(std::vector<TableSchema> &table_schema_array) {
 
             table_schema.dimension_ = resRow["dimension"];
 
+            table_schema.index_file_size_ = resRow["index_file_size"];
+
             table_schema.engine_type_ = resRow["engine_type"];
 
             table_schema.nlist_ = resRow["nlist"];
 
-            table_schema.index_file_size_ = resRow["index_file_size"];
-
             table_schema.metric_type_ = resRow["metric_type"];
 
             table_schema_array.emplace_back(table_schema);
@@ -833,9 +831,9 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
         file_schema.row_count_ = 0;
         file_schema.created_on_ = utils::GetMicroSecTimeStamp();
         file_schema.updated_time_ = file_schema.created_on_;
+        file_schema.index_file_size_ = table_schema.index_file_size_;
         file_schema.engine_type_ = table_schema.engine_type_;
         file_schema.nlist_ = table_schema.nlist_;
-        file_schema.index_file_size_ = table_schema.index_file_size_;
         file_schema.metric_type_ = table_schema.metric_type_;
         utils::GetTableFilePath(options_, file_schema);
 
@@ -950,10 +948,10 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) {
                 groups[table_file.table_id_] = table_schema;
 
             }
-            table_file.nlist_ = groups[table_file.table_id_].nlist_;
+            table_file.dimension_ = groups[table_file.table_id_].dimension_;
             table_file.index_file_size_ = groups[table_file.table_id_].index_file_size_;
+            table_file.nlist_ = groups[table_file.table_id_].nlist_;
             table_file.metric_type_ = groups[table_file.table_id_].metric_type_;
-            table_file.dimension_ = groups[table_file.table_id_].dimension_;
 
             utils::GetTableFilePath(options_, table_file);
 
@@ -1043,12 +1041,12 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id,
             resRow["table_id"].to_string(table_id_str);
             table_file.table_id_ = table_id_str;
 
+            table_file.index_file_size_ = table_schema.index_file_size_;
+
             table_file.engine_type_ = resRow["engine_type"];
 
             table_file.nlist_ = table_schema.nlist_;
 
-            table_file.index_file_size_ = table_schema.index_file_size_;
-
             table_file.metric_type_ = table_schema.metric_type_;
 
             std::string file_id;
@@ -1157,12 +1155,12 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id,
             resRow["table_id"].to_string(table_id_str);
             table_file.table_id_ = table_id_str;
 
+            table_file.index_file_size_ = table_schema.index_file_size_;
+
             table_file.engine_type_ = resRow["engine_type"];
 
             table_file.nlist_ = table_schema.nlist_;
 
-            table_file.index_file_size_ = table_schema.index_file_size_;
-
             table_file.metric_type_ = table_schema.metric_type_;
 
             std::string file_id;
@@ -1259,12 +1257,12 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id,
 
             table_file.date_ = resRow["date"];
 
+            table_file.index_file_size_ = table_schema.index_file_size_;
+
             table_file.engine_type_ = resRow["engine_type"];
 
             table_file.nlist_ = table_schema.nlist_;
 
-            table_file.index_file_size_ = table_schema.index_file_size_;
-
             table_file.metric_type_ = table_schema.metric_type_;
 
             table_file.created_on_ = resRow["created_on"];
@@ -1344,12 +1342,12 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id,
 
             file_schema.table_id_ = table_id;
 
+            file_schema.index_file_size_ = table_schema.index_file_size_;
+
             file_schema.engine_type_ = resRow["engine_type"];
 
             file_schema.nlist_ = table_schema.nlist_;
 
-            file_schema.index_file_size_ = table_schema.index_file_size_;
-
             file_schema.metric_type_ = table_schema.metric_type_;
 
             std::string file_id;

cpp/src/db/meta/SqliteMetaImpl.cpp

@@ -45,9 +45,9 @@ inline auto StoragePrototype(const std::string &path) {
                                    make_column("dimension", &TableSchema::dimension_),
                                    make_column("created_on", &TableSchema::created_on_),
                                    make_column("flag", &TableSchema::flag_, default_value(0)),
+                                   make_column("index_file_size", &TableSchema::index_file_size_),
                                    make_column("engine_type", &TableSchema::engine_type_),
                                    make_column("nlist", &TableSchema::nlist_),
-                                   make_column("index_file_size", &TableSchema::index_file_size_),
                                    make_column("metric_type", &TableSchema::metric_type_)),
                         make_table("TableFiles",
                                    make_column("id", &TableFileSchema::id_, primary_key()),
@@ -251,9 +251,9 @@ Status SqliteMetaImpl::DescribeTable(TableSchema &table_schema) {
                                                    &TableSchema::dimension_,
                                                    &TableSchema::created_on_,
                                                    &TableSchema::flag_,
+                                                   &TableSchema::index_file_size_,
                                                    &TableSchema::engine_type_,
                                                    &TableSchema::nlist_,
-                                                   &TableSchema::index_file_size_,
                                                    &TableSchema::metric_type_),
                                            where(c(&TableSchema::table_id_) == table_schema.table_id_
                                                  and c(&TableSchema::state_) != (int)TableSchema::TO_DELETE));
@@ -264,9 +264,9 @@ Status SqliteMetaImpl::DescribeTable(TableSchema &table_schema) {
             table_schema.dimension_ = std::get<2>(groups[0]);
             table_schema.created_on_ = std::get<3>(groups[0]);
             table_schema.flag_ = std::get<4>(groups[0]);
-            table_schema.engine_type_ = std::get<5>(groups[0]);
-            table_schema.nlist_ = std::get<6>(groups[0]);
-            table_schema.index_file_size_ = std::get<7>(groups[0]);
+            table_schema.index_file_size_ = std::get<5>(groups[0]);
+            table_schema.engine_type_ = std::get<6>(groups[0]);
+            table_schema.nlist_ = std::get<7>(groups[0]);
             table_schema.metric_type_ = std::get<8>(groups[0]);
         } else {
             return Status::NotFound("Table " + table_schema.table_id_ + " not found");
@@ -344,7 +344,8 @@ Status SqliteMetaImpl::UpdateTableIndexParam(const std::string &table_id, const
                                                    &TableSchema::state_,
                                                    &TableSchema::dimension_,
                                                    &TableSchema::created_on_,
-                                                   &TableSchema::flag_),
+                                                   &TableSchema::flag_,
+                                                   &TableSchema::index_file_size_),
                                            where(c(&TableSchema::table_id_) == table_id
                                                  and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE));
 
@@ -356,9 +357,9 @@ Status SqliteMetaImpl::UpdateTableIndexParam(const std::string &table_id, const
             table_schema.dimension_ = std::get<2>(tables[0]);
             table_schema.created_on_ = std::get<3>(tables[0]);
             table_schema.flag_ = std::get<4>(tables[0]);
+            table_schema.index_file_size_ = std::get<5>(tables[0]);
             table_schema.engine_type_ = index.engine_type_;
             table_schema.nlist_ = index.nlist_;
-            table_schema.index_file_size_ = index.index_file_size_*ONE_MB;
             table_schema.metric_type_ = index.metric_type_;
 
             ConnectorPtr->update(table_schema);
@@ -420,8 +421,7 @@ Status SqliteMetaImpl::DescribeTableIndex(const std::string &table_id, TableInde
         if (groups.size() == 1) {
             index.engine_type_ = std::get<0>(groups[0]);
             index.nlist_ = std::get<1>(groups[0]);
-            index.index_file_size_ = std::get<2>(groups[0])/ONE_MB;
-            index.metric_type_ = std::get<3>(groups[0]);
+            index.metric_type_ = std::get<2>(groups[0]);
         } else {
             return Status::NotFound("Table " + table_id + " not found");
         }
@@ -499,9 +499,9 @@ Status SqliteMetaImpl::AllTables(std::vector<TableSchema>& table_schema_array) {
                                                      &TableSchema::dimension_,
                                                      &TableSchema::created_on_,
                                                      &TableSchema::flag_,
+                                                     &TableSchema::index_file_size_,
                                                      &TableSchema::engine_type_,
                                                      &TableSchema::nlist_,
-                                                     &TableSchema::index_file_size_,
                                                      &TableSchema::metric_type_),
                                              where(c(&TableSchema::state_) != (int)TableSchema::TO_DELETE));
         for (auto &table : selected) {
@@ -511,9 +511,9 @@ Status SqliteMetaImpl::AllTables(std::vector<TableSchema>& table_schema_array) {
             schema.dimension_ = std::get<2>(table);
             schema.created_on_ = std::get<3>(table);
             schema.flag_ = std::get<4>(table);
-            schema.engine_type_ = std::get<5>(table);
-            schema.nlist_ = std::get<6>(table);
-            schema.index_file_size_ = std::get<7>(table);
+            schema.index_file_size_ = std::get<5>(table);
+            schema.engine_type_ = std::get<6>(table);
+            schema.nlist_ = std::get<7>(table);
             schema.metric_type_ = std::get<8>(table);
 
             table_schema_array.emplace_back(schema);
@@ -546,9 +546,9 @@ Status SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
         file_schema.row_count_ = 0;
         file_schema.created_on_ = utils::GetMicroSecTimeStamp();
         file_schema.updated_time_ = file_schema.created_on_;
+        file_schema.index_file_size_ = table_schema.index_file_size_;
         file_schema.engine_type_ = table_schema.engine_type_;
         file_schema.nlist_ = table_schema.nlist_;
-        file_schema.index_file_size_ = table_schema.index_file_size_;
         file_schema.metric_type_ = table_schema.metric_type_;
 
         //multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here
@@ -609,10 +609,10 @@ Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) {
                 }
                 groups[table_file.table_id_] = table_schema;
             }
-            table_file.nlist_ = groups[table_file.table_id_].nlist_;
+            table_file.dimension_ = groups[table_file.table_id_].dimension_;
             table_file.index_file_size_ = groups[table_file.table_id_].index_file_size_;
+            table_file.nlist_ = groups[table_file.table_id_].nlist_;
             table_file.metric_type_ = groups[table_file.table_id_].metric_type_;
-            table_file.dimension_ = groups[table_file.table_id_].dimension_;
             files.push_back(table_file);
         }
 
@@ -662,9 +662,11 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
                 table_file.row_count_ = std::get<5>(file);
                 table_file.date_ = std::get<6>(file);
                 table_file.engine_type_ = std::get<7>(file);
-                table_file.metric_type_ = table_schema.metric_type_;
-                table_file.nlist_ = table_schema.nlist_;
                 table_file.dimension_ = table_schema.dimension_;
+                table_file.index_file_size_ = table_schema.index_file_size_;
+                table_file.nlist_ = table_schema.nlist_;
+                table_file.metric_type_ = table_schema.metric_type_;
+
                 utils::GetTableFilePath(options_, table_file);
                 auto dateItr = files.find(table_file.date_);
                 if (dateItr == files.end()) {
@@ -705,10 +707,10 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
                 table_file.row_count_ = std::get<5>(file);
                 table_file.date_ = std::get<6>(file);
                 table_file.engine_type_ = std::get<7>(file);
-                table_file.nlist_ = table_schema.nlist_;
+                table_file.dimension_ = table_schema.dimension_;
                 table_file.index_file_size_ = table_schema.index_file_size_;
+                table_file.nlist_ = table_schema.nlist_;
                 table_file.metric_type_ = table_schema.metric_type_;
-                table_file.dimension_ = table_schema.dimension_;
 
                 utils::GetTableFilePath(options_, table_file);
                 auto dateItr = files.find(table_file.date_);
@@ -786,8 +788,8 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
             table_file.date_ = std::get<6>(file);
             table_file.engine_type_ = std::get<7>(file);
             table_file.dimension_ = table_schema.dimension_;
-            table_file.nlist_ = table_schema.nlist_;
             table_file.index_file_size_ = table_schema.index_file_size_;
+            table_file.nlist_ = table_schema.nlist_;
             table_file.metric_type_ = table_schema.metric_type_;
 
             utils::GetTableFilePath(options_, table_file);
@@ -848,8 +850,8 @@ Status SqliteMetaImpl::FilesToMerge(const std::string &table_id,
             table_file.date_ = std::get<6>(file);
             table_file.created_on_ = std::get<7>(file);
             table_file.dimension_ = table_schema.dimension_;
-            table_file.nlist_ = table_schema.nlist_;
             table_file.index_file_size_ = table_schema.index_file_size_;
+            table_file.nlist_ = table_schema.nlist_;
             table_file.metric_type_ = table_schema.metric_type_;
 
             utils::GetTableFilePath(options_, table_file);
@@ -900,11 +902,11 @@ Status SqliteMetaImpl::GetTableFiles(const std::string& table_id,
             file_schema.row_count_ = std::get<4>(file);
             file_schema.date_ = std::get<5>(file);
             file_schema.engine_type_ = std::get<6>(file);
-            file_schema.nlist_ = table_schema.nlist_;
-            file_schema.index_file_size_ = table_schema.index_file_size_;
-            file_schema.metric_type_ = table_schema.metric_type_;
             file_schema.created_on_ = std::get<7>(file);
             file_schema.dimension_ = table_schema.dimension_;
+            file_schema.index_file_size_ = table_schema.index_file_size_;
+            file_schema.nlist_ = table_schema.nlist_;
+            file_schema.metric_type_ = table_schema.metric_type_;
 
             utils::GetTableFilePath(options_, file_schema);
 

cpp/src/grpc/gen-milvus/milvus.pb.cc

@@ -366,6 +366,7 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT
   ~0u,  // no _weak_field_map_
   PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, table_name_),
   PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, dimension_),
+  PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, index_file_size_),
   ~0u,  // no _has_bits_
   PROTOBUF_FIELD_OFFSET(::milvus::grpc::Range, _internal_metadata_),
   ~0u,  // no _extensions_
@@ -459,7 +460,6 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT
   ~0u,  // no _weak_field_map_
   PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, index_type_),
   PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, nlist_),
-  PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, index_file_size_),
   PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, metric_type_),
   ~0u,  // no _has_bits_
   PROTOBUF_FIELD_OFFSET(::milvus::grpc::IndexParam, _internal_metadata_),
@@ -479,19 +479,19 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT
 static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = {
   { 0, -1, sizeof(::milvus::grpc::TableName)},
   { 7, -1, sizeof(::milvus::grpc::TableSchema)},
-  { 14, -1, sizeof(::milvus::grpc::Range)},
-  { 21, -1, sizeof(::milvus::grpc::RowRecord)},
-  { 27, -1, sizeof(::milvus::grpc::InsertParam)},
-  { 35, -1, sizeof(::milvus::grpc::VectorIds)},
-  { 42, -1, sizeof(::milvus::grpc::SearchParam)},
-  { 52, -1, sizeof(::milvus::grpc::SearchInFilesParam)},
-  { 59, -1, sizeof(::milvus::grpc::QueryResult)},
-  { 66, -1, sizeof(::milvus::grpc::TopKQueryResult)},
-  { 73, -1, sizeof(::milvus::grpc::StringReply)},
-  { 80, -1, sizeof(::milvus::grpc::BoolReply)},
-  { 87, -1, sizeof(::milvus::grpc::TableRowCount)},
-  { 94, -1, sizeof(::milvus::grpc::Command)},
-  { 100, -1, sizeof(::milvus::grpc::Index)},
+  { 15, -1, sizeof(::milvus::grpc::Range)},
+  { 22, -1, sizeof(::milvus::grpc::RowRecord)},
+  { 28, -1, sizeof(::milvus::grpc::InsertParam)},
+  { 36, -1, sizeof(::milvus::grpc::VectorIds)},
+  { 43, -1, sizeof(::milvus::grpc::SearchParam)},
+  { 53, -1, sizeof(::milvus::grpc::SearchInFilesParam)},
+  { 60, -1, sizeof(::milvus::grpc::QueryResult)},
+  { 67, -1, sizeof(::milvus::grpc::TopKQueryResult)},
+  { 74, -1, sizeof(::milvus::grpc::StringReply)},
+  { 81, -1, sizeof(::milvus::grpc::BoolReply)},
+  { 88, -1, sizeof(::milvus::grpc::TableRowCount)},
+  { 95, -1, sizeof(::milvus::grpc::Command)},
+  { 101, -1, sizeof(::milvus::grpc::Index)},
   { 109, -1, sizeof(::milvus::grpc::IndexParam)},
   { 116, -1, sizeof(::milvus::grpc::DeleteByRangeParam)},
 };
@@ -519,35 +519,35 @@ static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] =
 const char descriptor_table_protodef_milvus_2eproto[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) =
   "\n\014milvus.proto\022\013milvus.grpc\032\014status.prot"
   "o\"D\n\tTableName\022#\n\006status\030\001 \001(\0132\023.milvus."
-  "grpc.Status\022\022\n\ntable_name\030\002 \001(\t\"L\n\013Table"
+  "grpc.Status\022\022\n\ntable_name\030\002 \001(\t\"e\n\013Table"
   "Schema\022*\n\ntable_name\030\001 \001(\0132\026.milvus.grpc"
-  ".TableName\022\021\n\tdimension\030\002 \001(\003\"/\n\005Range\022\023"
-  "\n\013start_value\030\001 \001(\t\022\021\n\tend_value\030\002 \001(\t\" "
-  "\n\tRowRecord\022\023\n\013vector_data\030\001 \003(\002\"i\n\013Inse"
-  "rtParam\022\022\n\ntable_name\030\001 \001(\t\0220\n\020row_recor"
-  "d_array\030\002 \003(\0132\026.milvus.grpc.RowRecord\022\024\n"
-  "\014row_id_array\030\003 \003(\003\"I\n\tVectorIds\022#\n\006stat"
-  "us\030\001 \001(\0132\023.milvus.grpc.Status\022\027\n\017vector_"
-  "id_array\030\002 \003(\003\"\242\001\n\013SearchParam\022\022\n\ntable_"
-  "name\030\001 \001(\t\0222\n\022query_record_array\030\002 \003(\0132\026"
-  ".milvus.grpc.RowRecord\022-\n\021query_range_ar"
-  "ray\030\003 \003(\0132\022.milvus.grpc.Range\022\014\n\004topk\030\004 "
-  "\001(\003\022\016\n\006nprobe\030\005 \001(\003\"[\n\022SearchInFilesPara"
-  "m\022\025\n\rfile_id_array\030\001 \003(\t\022.\n\014search_param"
-  "\030\002 \001(\0132\030.milvus.grpc.SearchParam\"+\n\013Quer"
-  "yResult\022\n\n\002id\030\001 \001(\003\022\020\n\010distance\030\002 \001(\001\"m\n"
-  "\017TopKQueryResult\022#\n\006status\030\001 \001(\0132\023.milvu"
-  "s.grpc.Status\0225\n\023query_result_arrays\030\002 \003"
-  "(\0132\030.milvus.grpc.QueryResult\"H\n\013StringRe"
-  "ply\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.Status"
-  "\022\024\n\014string_reply\030\002 \001(\t\"D\n\tBoolReply\022#\n\006s"
-  "tatus\030\001 \001(\0132\023.milvus.grpc.Status\022\022\n\nbool"
-  "_reply\030\002 \001(\010\"M\n\rTableRowCount\022#\n\006status\030"
-  "\001 \001(\0132\023.milvus.grpc.Status\022\027\n\017table_row_"
-  "count\030\002 \001(\003\"\026\n\007Command\022\013\n\003cmd\030\001 \001(\t\"X\n\005I"
-  "ndex\022\022\n\nindex_type\030\001 \001(\005\022\r\n\005nlist\030\002 \001(\005\022"
-  "\027\n\017index_file_size\030\003 \001(\005\022\023\n\013metric_type\030"
-  "\004 \001(\005\"[\n\nIndexParam\022*\n\ntable_name\030\001 \001(\0132"
+  ".TableName\022\021\n\tdimension\030\002 \001(\003\022\027\n\017index_f"
+  "ile_size\030\003 \001(\003\"/\n\005Range\022\023\n\013start_value\030\001"
+  " \001(\t\022\021\n\tend_value\030\002 \001(\t\" \n\tRowRecord\022\023\n\013"
+  "vector_data\030\001 \003(\002\"i\n\013InsertParam\022\022\n\ntabl"
+  "e_name\030\001 \001(\t\0220\n\020row_record_array\030\002 \003(\0132\026"
+  ".milvus.grpc.RowRecord\022\024\n\014row_id_array\030\003"
+  " \003(\003\"I\n\tVectorIds\022#\n\006status\030\001 \001(\0132\023.milv"
+  "us.grpc.Status\022\027\n\017vector_id_array\030\002 \003(\003\""
+  "\242\001\n\013SearchParam\022\022\n\ntable_name\030\001 \001(\t\0222\n\022q"
+  "uery_record_array\030\002 \003(\0132\026.milvus.grpc.Ro"
+  "wRecord\022-\n\021query_range_array\030\003 \003(\0132\022.mil"
+  "vus.grpc.Range\022\014\n\004topk\030\004 \001(\003\022\016\n\006nprobe\030\005"
+  " \001(\003\"[\n\022SearchInFilesParam\022\025\n\rfile_id_ar"
+  "ray\030\001 \003(\t\022.\n\014search_param\030\002 \001(\0132\030.milvus"
+  ".grpc.SearchParam\"+\n\013QueryResult\022\n\n\002id\030\001"
+  " \001(\003\022\020\n\010distance\030\002 \001(\001\"m\n\017TopKQueryResul"
+  "t\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.Status\0225"
+  "\n\023query_result_arrays\030\002 \003(\0132\030.milvus.grp"
+  "c.QueryResult\"H\n\013StringReply\022#\n\006status\030\001"
+  " \001(\0132\023.milvus.grpc.Status\022\024\n\014string_repl"
+  "y\030\002 \001(\t\"D\n\tBoolReply\022#\n\006status\030\001 \001(\0132\023.m"
+  "ilvus.grpc.Status\022\022\n\nbool_reply\030\002 \001(\010\"M\n"
+  "\rTableRowCount\022#\n\006status\030\001 \001(\0132\023.milvus."
+  "grpc.Status\022\027\n\017table_row_count\030\002 \001(\003\"\026\n\007"
+  "Command\022\013\n\003cmd\030\001 \001(\t\"\?\n\005Index\022\022\n\nindex_t"
+  "ype\030\001 \001(\005\022\r\n\005nlist\030\002 \001(\005\022\023\n\013metric_type\030"
+  "\003 \001(\005\"[\n\nIndexParam\022*\n\ntable_name\030\001 \001(\0132"
   "\026.milvus.grpc.TableName\022!\n\005index\030\002 \001(\0132\022"
   ".milvus.grpc.Index\"K\n\022DeleteByRangeParam"
   "\022!\n\005range\030\001 \001(\0132\022.milvus.grpc.Range\022\022\n\nt"
@@ -978,15 +978,17 @@ TableSchema::TableSchema(const TableSchema& from)
   } else {
     table_name_ = nullptr;
   }
-  dimension_ = from.dimension_;
+  ::memcpy(&dimension_, &from.dimension_,
+    static_cast<size_t>(reinterpret_cast<char*>(&index_file_size_) -
+    reinterpret_cast<char*>(&dimension_)) + sizeof(index_file_size_));
   // @@protoc_insertion_point(copy_constructor:milvus.grpc.TableSchema)
 }
 
 void TableSchema::SharedCtor() {
   ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_TableSchema_milvus_2eproto.base);
   ::memset(&table_name_, 0, static_cast<size_t>(
-      reinterpret_cast<char*>(&dimension_) -
-      reinterpret_cast<char*>(&table_name_)) + sizeof(dimension_));
+      reinterpret_cast<char*>(&index_file_size_) -
+      reinterpret_cast<char*>(&table_name_)) + sizeof(index_file_size_));
 }
 
 TableSchema::~TableSchema() {
@@ -1017,7 +1019,9 @@ void TableSchema::Clear() {
     delete table_name_;
   }
   table_name_ = nullptr;
-  dimension_ = PROTOBUF_LONGLONG(0);
+  ::memset(&dimension_, 0, static_cast<size_t>(
+      reinterpret_cast<char*>(&index_file_size_) -
+      reinterpret_cast<char*>(&dimension_)) + sizeof(index_file_size_));
   _internal_metadata_.Clear();
 }
 
@@ -1043,6 +1047,13 @@ const char* TableSchema::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID
           CHK_(ptr);
         } else goto handle_unusual;
         continue;
+      // int64 index_file_size = 3;
+      case 3:
+        if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) {
+          index_file_size_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr);
+          CHK_(ptr);
+        } else goto handle_unusual;
+        continue;
       default: {
       handle_unusual:
         if ((tag & 7) == 4 || tag == 0) {
@@ -1097,6 +1108,19 @@ bool TableSchema::MergePartialFromCodedStream(
         break;
       }
 
+      // int64 index_file_size = 3;
+      case 3: {
+        if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (24 & 0xFF)) {
+
+          DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive<
+                   ::PROTOBUF_NAMESPACE_ID::int64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT64>(
+                 input, &index_file_size_)));
+        } else {
+          goto handle_unusual;
+        }
+        break;
+      }
+
       default: {
       handle_unusual:
         if (tag == 0) {
@@ -1135,6 +1159,11 @@ void TableSchema::SerializeWithCachedSizes(
     ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(2, this->dimension(), output);
   }
 
+  // int64 index_file_size = 3;
+  if (this->index_file_size() != 0) {
+    ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(3, this->index_file_size(), output);
+  }
+
   if (_internal_metadata_.have_unknown_fields()) {
     ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields(
         _internal_metadata_.unknown_fields(), output);
@@ -1160,6 +1189,11 @@ void TableSchema::SerializeWithCachedSizes(
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(2, this->dimension(), target);
   }
 
+  // int64 index_file_size = 3;
+  if (this->index_file_size() != 0) {
+    target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(3, this->index_file_size(), target);
+  }
+
   if (_internal_metadata_.have_unknown_fields()) {
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray(
         _internal_metadata_.unknown_fields(), target);
@@ -1195,6 +1229,13 @@ size_t TableSchema::ByteSizeLong() const {
         this->dimension());
   }
 
+  // int64 index_file_size = 3;
+  if (this->index_file_size() != 0) {
+    total_size += 1 +
+      ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int64Size(
+        this->index_file_size());
+  }
+
   int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size);
   SetCachedSize(cached_size);
   return total_size;
@@ -1228,6 +1269,9 @@ void TableSchema::MergeFrom(const TableSchema& from) {
   if (from.dimension() != 0) {
     set_dimension(from.dimension());
   }
+  if (from.index_file_size() != 0) {
+    set_index_file_size(from.index_file_size());
+  }
 }
 
 void TableSchema::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) {
@@ -1253,6 +1297,7 @@ void TableSchema::InternalSwap(TableSchema* other) {
   _internal_metadata_.Swap(&other->_internal_metadata_);
   swap(table_name_, other->table_name_);
   swap(dimension_, other->dimension_);
+  swap(index_file_size_, other->index_file_size_);
 }
 
 ::PROTOBUF_NAMESPACE_ID::Metadata TableSchema::GetMetadata() const {
@@ -5331,16 +5376,9 @@ const char* Index::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::inte
           CHK_(ptr);
         } else goto handle_unusual;
         continue;
-      // int32 index_file_size = 3;
+      // int32 metric_type = 3;
       case 3:
         if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) {
-          index_file_size_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr);
-          CHK_(ptr);
-        } else goto handle_unusual;
-        continue;
-      // int32 metric_type = 4;
-      case 4:
-        if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 32)) {
           metric_type_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr);
           CHK_(ptr);
         } else goto handle_unusual;
@@ -5401,23 +5439,10 @@ bool Index::MergePartialFromCodedStream(
         break;
       }
 
-      // int32 index_file_size = 3;
+      // int32 metric_type = 3;
       case 3: {
         if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (24 & 0xFF)) {
 
-          DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive<
-                   ::PROTOBUF_NAMESPACE_ID::int32, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT32>(
-                 input, &index_file_size_)));
-        } else {
-          goto handle_unusual;
-        }
-        break;
-      }
-
-      // int32 metric_type = 4;
-      case 4: {
-        if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (32 & 0xFF)) {
-
           DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive<
                    ::PROTOBUF_NAMESPACE_ID::int32, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT32>(
                  input, &metric_type_)));
@@ -5464,14 +5489,9 @@ void Index::SerializeWithCachedSizes(
     ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(2, this->nlist(), output);
   }
 
-  // int32 index_file_size = 3;
-  if (this->index_file_size() != 0) {
-    ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(3, this->index_file_size(), output);
-  }
-
-  // int32 metric_type = 4;
+  // int32 metric_type = 3;
   if (this->metric_type() != 0) {
-    ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(4, this->metric_type(), output);
+    ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(3, this->metric_type(), output);
   }
 
   if (_internal_metadata_.have_unknown_fields()) {
@@ -5497,14 +5517,9 @@ void Index::SerializeWithCachedSizes(
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(2, this->nlist(), target);
   }
 
-  // int32 index_file_size = 3;
-  if (this->index_file_size() != 0) {
-    target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(3, this->index_file_size(), target);
-  }
-
-  // int32 metric_type = 4;
+  // int32 metric_type = 3;
   if (this->metric_type() != 0) {
-    target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(4, this->metric_type(), target);
+    target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(3, this->metric_type(), target);
   }
 
   if (_internal_metadata_.have_unknown_fields()) {
@@ -5542,14 +5557,7 @@ size_t Index::ByteSizeLong() const {
         this->nlist());
   }
 
-  // int32 index_file_size = 3;
-  if (this->index_file_size() != 0) {
-    total_size += 1 +
-      ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(
-        this->index_file_size());
-  }
-
-  // int32 metric_type = 4;
+  // int32 metric_type = 3;
   if (this->metric_type() != 0) {
     total_size += 1 +
       ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(
@@ -5589,9 +5597,6 @@ void Index::MergeFrom(const Index& from) {
   if (from.nlist() != 0) {
     set_nlist(from.nlist());
   }
-  if (from.index_file_size() != 0) {
-    set_index_file_size(from.index_file_size());
-  }
   if (from.metric_type() != 0) {
     set_metric_type(from.metric_type());
   }
@@ -5620,7 +5625,6 @@ void Index::InternalSwap(Index* other) {
   _internal_metadata_.Swap(&other->_internal_metadata_);
   swap(index_type_, other->index_type_);
   swap(nlist_, other->nlist_);
-  swap(index_file_size_, other->index_file_size_);
   swap(metric_type_, other->metric_type_);
 }
 

cpp/src/grpc/gen-milvus/milvus.pb.h

@@ -396,6 +396,7 @@ class TableSchema :
   enum : int {
     kTableNameFieldNumber = 1,
     kDimensionFieldNumber = 2,
+    kIndexFileSizeFieldNumber = 3,
   };
   // .milvus.grpc.TableName table_name = 1;
   bool has_table_name() const;
@@ -410,6 +411,11 @@ class TableSchema :
   ::PROTOBUF_NAMESPACE_ID::int64 dimension() const;
   void set_dimension(::PROTOBUF_NAMESPACE_ID::int64 value);
 
+  // int64 index_file_size = 3;
+  void clear_index_file_size();
+  ::PROTOBUF_NAMESPACE_ID::int64 index_file_size() const;
+  void set_index_file_size(::PROTOBUF_NAMESPACE_ID::int64 value);
+
   // @@protoc_insertion_point(class_scope:milvus.grpc.TableSchema)
  private:
   class _Internal;
@@ -417,6 +423,7 @@ class TableSchema :
   ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_;
   ::milvus::grpc::TableName* table_name_;
   ::PROTOBUF_NAMESPACE_ID::int64 dimension_;
+  ::PROTOBUF_NAMESPACE_ID::int64 index_file_size_;
   mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
   friend struct ::TableStruct_milvus_2eproto;
 };
@@ -2318,8 +2325,7 @@ class Index :
   enum : int {
     kIndexTypeFieldNumber = 1,
     kNlistFieldNumber = 2,
-    kIndexFileSizeFieldNumber = 3,
-    kMetricTypeFieldNumber = 4,
+    kMetricTypeFieldNumber = 3,
   };
   // int32 index_type = 1;
   void clear_index_type();
@@ -2331,12 +2337,7 @@ class Index :
   ::PROTOBUF_NAMESPACE_ID::int32 nlist() const;
   void set_nlist(::PROTOBUF_NAMESPACE_ID::int32 value);
 
-  // int32 index_file_size = 3;
-  void clear_index_file_size();
-  ::PROTOBUF_NAMESPACE_ID::int32 index_file_size() const;
-  void set_index_file_size(::PROTOBUF_NAMESPACE_ID::int32 value);
-
-  // int32 metric_type = 4;
+  // int32 metric_type = 3;
   void clear_metric_type();
   ::PROTOBUF_NAMESPACE_ID::int32 metric_type() const;
   void set_metric_type(::PROTOBUF_NAMESPACE_ID::int32 value);
@@ -2348,7 +2349,6 @@ class Index :
   ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_;
   ::PROTOBUF_NAMESPACE_ID::int32 index_type_;
   ::PROTOBUF_NAMESPACE_ID::int32 nlist_;
-  ::PROTOBUF_NAMESPACE_ID::int32 index_file_size_;
   ::PROTOBUF_NAMESPACE_ID::int32 metric_type_;
   mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
   friend struct ::TableStruct_milvus_2eproto;
@@ -2820,6 +2820,20 @@ inline void TableSchema::set_dimension(::PROTOBUF_NAMESPACE_ID::int64 value) {
   // @@protoc_insertion_point(field_set:milvus.grpc.TableSchema.dimension)
 }
 
+// int64 index_file_size = 3;
+inline void TableSchema::clear_index_file_size() {
+  index_file_size_ = PROTOBUF_LONGLONG(0);
+}
+inline ::PROTOBUF_NAMESPACE_ID::int64 TableSchema::index_file_size() const {
+  // @@protoc_insertion_point(field_get:milvus.grpc.TableSchema.index_file_size)
+  return index_file_size_;
+}
+inline void TableSchema::set_index_file_size(::PROTOBUF_NAMESPACE_ID::int64 value) {
+  
+  index_file_size_ = value;
+  // @@protoc_insertion_point(field_set:milvus.grpc.TableSchema.index_file_size)
+}
+
 // -------------------------------------------------------------------
 
 // Range
@@ -3841,21 +3855,7 @@ inline void Index::set_nlist(::PROTOBUF_NAMESPACE_ID::int32 value) {
   // @@protoc_insertion_point(field_set:milvus.grpc.Index.nlist)
 }
 
-// int32 index_file_size = 3;
-inline void Index::clear_index_file_size() {
-  index_file_size_ = 0;
-}
-inline ::PROTOBUF_NAMESPACE_ID::int32 Index::index_file_size() const {
-  // @@protoc_insertion_point(field_get:milvus.grpc.Index.index_file_size)
-  return index_file_size_;
-}
-inline void Index::set_index_file_size(::PROTOBUF_NAMESPACE_ID::int32 value) {
-  
-  index_file_size_ = value;
-  // @@protoc_insertion_point(field_set:milvus.grpc.Index.index_file_size)
-}
-
-// int32 metric_type = 4;
+// int32 metric_type = 3;
 inline void Index::clear_metric_type() {
   metric_type_ = 0;
 }

cpp/src/grpc/milvus.proto

@@ -18,6 +18,7 @@ message TableName {
 message TableSchema {
     TableName table_name = 1;
     int64 dimension = 2;
+    int64 index_file_size = 3;
 }
 
 /**
@@ -126,8 +127,7 @@ message Command {
 message Index {
     int32 index_type = 1;
     int32 nlist = 2;
-    int32 index_file_size = 3;
-    int32 metric_type = 4;
+    int32 metric_type = 3;
 }
 
 /**

cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp

@@ -22,6 +22,7 @@ namespace {
 
     const std::string TABLE_NAME = GetTableName();
     constexpr int64_t TABLE_DIMENSION = 512;
+    constexpr int64_t TABLE_INDEX_FILE_SIZE = 768;
     constexpr int64_t BATCH_ROW_COUNT = 100000;
     constexpr int64_t NQ = 10;
     constexpr int64_t TOP_K = 10;
@@ -94,6 +95,7 @@ namespace {
         TableSchema tb_schema;
         tb_schema.table_name = TABLE_NAME;
         tb_schema.dimension = TABLE_DIMENSION;
+        tb_schema.index_file_size = TABLE_INDEX_FILE_SIZE;
 
         return tb_schema;
     }
@@ -286,7 +288,6 @@ ClientTest::Test(const std::string& address, const std::string& port) {
         index.table_name = TABLE_NAME;
         index.index_type = IndexType::gpu_ivfflat;
         index.nlist = 1000;
-        index.index_file_size = 1024;
         index.metric_type = 1;
         Status stat = conn->CreateIndex(index);
         std::cout << "CreateIndex function call status: " << stat.ToString() << std::endl;

cpp/src/sdk/grpc/ClientProxy.cpp

@@ -83,6 +83,7 @@ ClientProxy::CreateTable(const TableSchema &param) {
         ::milvus::grpc::TableSchema schema;
         schema.mutable_table_name()->set_table_name(param.table_name);
         schema.set_dimension(param.dimension);
+        schema.set_index_file_size(param.index_file_size);
 
         return client_ptr_->CreateTable(schema);
     } catch (std::exception &ex) {
@@ -119,7 +120,6 @@ ClientProxy::CreateIndex(const IndexParam &index_param) {
                 index_param.table_name);
         grpc_index_param.mutable_index()->set_index_type((int32_t)index_param.index_type);
         grpc_index_param.mutable_index()->set_nlist(index_param.nlist);
-        grpc_index_param.mutable_index()->set_index_file_size(index_param.index_file_size);
         grpc_index_param.mutable_index()->set_metric_type(index_param.metric_type);
         return client_ptr_->CreateIndex(grpc_index_param);
 
@@ -272,6 +272,7 @@ ClientProxy::DescribeTable(const std::string &table_name, TableSchema &table_sch
 
         table_schema.table_name = grpc_schema.table_name().table_name();
         table_schema.dimension = grpc_schema.dimension();
+        table_schema.index_file_size = grpc_schema.index_file_size();
 
         return status;
     } catch (std::exception &ex) {
@@ -362,7 +363,6 @@ ClientProxy::DescribeIndex(const std::string &table_name, IndexParam &index_para
         Status status = client_ptr_->DescribeIndex(grpc_table_name, grpc_index_param);
         index_param.index_type = (IndexType)(grpc_index_param.mutable_index()->index_type());
         index_param.nlist = grpc_index_param.mutable_index()->nlist();
-        index_param.index_file_size = grpc_index_param.mutable_index()->index_file_size();
         index_param.metric_type = grpc_index_param.mutable_index()->metric_type();
 
         return status;

cpp/src/sdk/include/MilvusApi.h

@@ -35,9 +35,8 @@ struct ConnectParam {
  */
 struct TableSchema {
     std::string table_name;                                ///< Table name
-    IndexType index_type = IndexType::invalid;             ///< Index type
     int64_t dimension = 0;                                 ///< Vector dimension, must be a positive value
-    bool store_raw_vector = false;                          ///< Is vector raw data stored in the table
+    int64_t index_file_size = 0;                           ///< Index file size, must be a positive value
 };
 
 /**
@@ -78,7 +77,6 @@ struct IndexParam {
     std::string table_name;
     IndexType index_type;
     int32_t nlist;
-    int32_t index_file_size;
     int32_t metric_type;
 };
 

cpp/src/server/grpc_impl/GrpcRequestTask.cpp

@@ -134,10 +134,16 @@ CreateTableTask::OnExecute() {
             return SetError(res, "Invalid table dimension: " + std::to_string(schema_.dimension()));
         }
 
+        res = ValidationUtil::ValidateTableIndexFileSize(schema_.index_file_size());
+        if(res != SERVER_SUCCESS) {
+            return SetError(res, "Invalid index file size: " + std::to_string(schema_.index_file_size()));
+        }
+
         //step 2: construct table schema
         engine::meta::TableSchema table_info;
-        table_info.dimension_ = (uint16_t) schema_.dimension();
         table_info.table_id_ = schema_.table_name().table_name();
+        table_info.dimension_ = (uint16_t) schema_.dimension();
+        table_info.index_file_size_ = schema_.index_file_size();
 
         //step 3: create table
         engine::Status stat = DBWrapper::DB()->CreateTable(table_info);
@@ -246,16 +252,10 @@ CreateIndexTask::OnExecute() {
             return SetError(res, "Invalid index metric type: " + std::to_string(index_param_.mutable_index()->metric_type()));
         }
 
-        res = ValidationUtil::ValidateTableIndexFileSize(index_param_.mutable_index()->index_file_size());
-        if(res != SERVER_SUCCESS) {
-            return SetError(res, "Invalid index file size: " + std::to_string(index_param_.mutable_index()->index_file_size()));
-        }
-
         //step 2: check table existence
         engine::TableIndex index;
         index.engine_type_ = index_param_.mutable_index()->index_type();
         index.nlist_ = index_param_.mutable_index()->nlist();
-        index.index_file_size_ = index_param_.mutable_index()->index_file_size();
         index.metric_type_ = index_param_.mutable_index()->metric_type();
         stat = DBWrapper::DB()->CreateIndex(table_name_, index);
         if (!stat.ok()) {
@@ -904,7 +904,6 @@ DescribeIndexTask::OnExecute() {
         index_param_.mutable_table_name()->set_table_name(table_name_);
         index_param_.mutable_index()->set_index_type(index.engine_type_);
         index_param_.mutable_index()->set_nlist(index.nlist_);
-        index_param_.mutable_index()->set_index_file_size(index.index_file_size_);
         index_param_.mutable_index()->set_metric_type(index.metric_type_);
 
         rc.ElapseFromBegin("totally cost");

cpp/src/utils/ValidationUtil.cpp

@@ -76,7 +76,7 @@ ValidationUtil::ValidateTableIndexNlist(int32_t nlist) {
 }
 
 ServerError
-ValidationUtil::ValidateTableIndexFileSize(int32_t index_file_size) {
+ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) {
     if(index_file_size <= 0 || index_file_size > index_file_size_limit) {
         return SERVER_INVALID_INDEX_FILE_SIZE;
     }

cpp/src/utils/ValidationUtil.h

@@ -21,7 +21,7 @@ public:
     ValidateTableIndexNlist(int32_t nlist);
 
     static ServerError
-    ValidateTableIndexFileSize(int32_t index_file_size);
+    ValidateTableIndexFileSize(int64_t index_file_size);
 
     static ServerError
     ValidateTableIndexMetricType(int32_t metric_type);