diff --git a/ci/jenkinsfile/cleanup_staging.groovy b/ci/jenkinsfile/cleanup_staging.groovy new file mode 100644 index 0000000000000000000000000000000000000000..2e9332fa6e623b07e74837d5e5feabed09bd3ce5 --- /dev/null +++ b/ci/jenkinsfile/cleanup_staging.groovy @@ -0,0 +1,13 @@ +try { + def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true + if (!result) { + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" + } +} catch (exc) { + def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true + if (!result) { + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" + } + throw exc +} + diff --git a/ci/jenkinsfile/deploy2staging.groovy b/ci/jenkinsfile/deploy2staging.groovy new file mode 100644 index 0000000000000000000000000000000000000000..8079698894a076d8038a7d7b44f41184007158d1 --- /dev/null +++ b/ci/jenkinsfile/deploy2staging.groovy @@ -0,0 +1,16 @@ +try { + sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' + sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus' + sh 'helm repo update' + dir ("milvus-helm") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + dir ("milvus/milvus-gpu") { + sh "helm install --wait --timeout 300 --set engine.image.repository="zilliz.azurecr.cn/milvus/engine" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.4.0 ." + } + } +} catch (exc) { + echo 'Helm running failed!' + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" + throw exc +} + diff --git a/ci/jenkinsfile/dev_test.groovy b/ci/jenkinsfile/dev_test.groovy index fc3badae0c573451d3b9ecdfa053b10959ca2461..8ae7e4c47863d860807abfac31fd32e0130fdefe 100644 --- a/ci/jenkinsfile/dev_test.groovy +++ b/ci/jenkinsfile/dev_test.groovy @@ -3,7 +3,7 @@ timeout(time: 40, unit: 'MINUTES') { dir ("${PROJECT_NAME}_test") { checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) sh 'python3 -m pip install -r requirements.txt' - sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local" + sh "pytest . --alluredir="test_out/dev/single/sqlite" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local" } // mysql database backend test @@ -16,11 +16,11 @@ timeout(time: 40, unit: 'MINUTES') { } dir ("milvus-helm") { dir ("milvus/milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.3.1 ." + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.4.0 ." } } dir ("${PROJECT_NAME}_test") { - sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local" + sh "pytest . --alluredir="test_out/dev/single/mysql" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local" } } catch (exc) { echo 'Milvus Test Failed !' diff --git a/ci/jenkinsfile/staging_test.groovy b/ci/jenkinsfile/staging_test.groovy new file mode 100644 index 0000000000000000000000000000000000000000..39fceb15384cfcae7a9097815d69454373e8c543 --- /dev/null +++ b/ci/jenkinsfile/staging_test.groovy @@ -0,0 +1,31 @@ +timeout(time: 40, unit: 'MINUTES') { + try { + dir ("${PROJECT_NAME}_test") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + sh 'python3 -m pip install -r requirements.txt' + def service_ip = sh (script: "kubectl get svc --namespace milvus-1 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine --template \"{{range .status.loadBalancer.ingress}}{{.ip}}{{end}}\"",returnStdout: true).trim() + sh "pytest . --alluredir="test_out/staging/single/sqlite" --ip ${service_ip}" + } + + // mysql database backend test + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy" + + if (!fileExists('milvus-helm')) { + dir ("milvus-helm") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + } + } + dir ("milvus-helm") { + dir ("milvus/milvus-gpu") { + sh "helm install --wait --timeout 300 --set engine.image.repository="zilliz.azurecr.cn/milvus/engine" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.4.0 ." + } + } + dir ("${PROJECT_NAME}_test") { + def service_ip = sh (script: "kubectl get svc --namespace milvus-2 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine --template \"{{range .status.loadBalancer.ingress}}{{.ip}}{{end}}\"",returnStdout: true).trim() + sh "pytest . --alluredir="test_out/staging/single/mysql" --ip ${service_ip}" + } + } catch (exc) { + echo 'Milvus Test Failed !' + throw exc + } +} diff --git a/ci/jenkinsfile/upload_dev_test_out.groovy b/ci/jenkinsfile/upload_dev_test_out.groovy index 7e106c0296a52c3330d29d933cea4b8f91ced5c5..017b887334b70bec10b9b34f0f7d69c8f9058e91 100644 --- a/ci/jenkinsfile/upload_dev_test_out.groovy +++ b/ci/jenkinsfile/upload_dev_test_out.groovy @@ -1,8 +1,8 @@ timeout(time: 5, unit: 'MINUTES') { dir ("${PROJECT_NAME}_test") { - if (fileExists('test_out')) { + if (fileExists('test_out/dev')) { def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" - fileTransfer.FileTransfer("test_out/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') + fileTransfer.FileTransfer("test_out/dev/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\"" } diff --git a/ci/jenkinsfile/upload_staging_test_out.groovy b/ci/jenkinsfile/upload_staging_test_out.groovy new file mode 100644 index 0000000000000000000000000000000000000000..1f1e66ab1bb383d461dad2ad798b455833ae326f --- /dev/null +++ b/ci/jenkinsfile/upload_staging_test_out.groovy @@ -0,0 +1,13 @@ +timeout(time: 5, unit: 'MINUTES') { + dir ("${PROJECT_NAME}_test") { + if (fileExists('test_out/staging')) { + def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" + fileTransfer.FileTransfer("test_out/staging/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') + if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\"" + } + } else { + error("Milvus Dev Test Out directory don't exists!") + } + } +} diff --git a/ci/nightly_main_jenkinsfile b/ci/nightly_main_jenkinsfile index e92d4f14f807c901103b4d70128c70d574f7d4b4..6f6773b6e7907a218607c6bf953db3d9bc63b85d 100644 --- a/ci/nightly_main_jenkinsfile +++ b/ci/nightly_main_jenkinsfile @@ -208,17 +208,17 @@ spec: } success { script { - echo "Milvus Single Node CI/CD success !" + echo "Milvus Deploy to Dev Single Node CI/CD success !" } } aborted { script { - echo "Milvus Single Node CI/CD aborted !" + echo "Milvus Deploy to Dev Single Node CI/CD aborted !" } } failure { script { - echo "Milvus Single Node CI/CD failure !" + echo "Milvus Deploy to Dev Single Node CI/CD failure !" } } } @@ -300,17 +300,114 @@ spec: } success { script { - echo "Milvus Cluster CI/CD success !" + echo "Milvus Deploy to Dev Cluster CI/CD success !" } } aborted { script { - echo "Milvus Cluster CI/CD aborted !" + echo "Milvus Deploy to Dev Cluster CI/CD aborted !" } } failure { script { - echo "Milvus Cluster CI/CD failure !" + echo "Milvus Deploy to Dev Cluster CI/CD failure !" + } + } + } + } + } + } + + stage("Deploy to Staging") { + parallel { + stage("Single Node") { + agent { + kubernetes { + label 'dev-test' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: test +spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.2 + command: + - cat + tty: true + volumeMounts: + - name: kubeconf + mountPath: /root/.kube/ + readOnly: true + volumes: + - name: kubeconf + secret: + secretName: aks-gpu-cluster-config +""" + } + } + + stages { + stage("Deploy to Staging") { + steps { + gitlabCommitStatus(name: 'Deloy to Staging') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/deploy2staging.groovy" + } + } + } + } + } + stage("Staging Test") { + steps { + gitlabCommitStatus(name: 'Staging Test') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/staging_test.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/upload_staging_test_out.groovy" + } + } + } + } + } + stage ("Cleanup Staging") { + steps { + gitlabCommitStatus(name: 'Cleanup Staging') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy" + } + } + } + } + } + } + post { + always { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy" + } + } + } + success { + script { + echo "Milvus Deploy to Staging Single Node CI/CD success !" + } + } + aborted { + script { + echo "Milvus Deploy to Staging Single Node CI/CD aborted !" + } + } + failure { + script { + echo "Milvus Deploy to Staging Single Node CI/CD failure !" } } } diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 1d7a7384d7baec0ede11e83fb3fc9edfdf25cd15..268940e8e34efd4f39be54416e548fb92823c193 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -56,6 +56,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-414 - Add TaskType in Scheduler::Task - MS-415 - Add command tasktable to dump all tasktables - MS-418 - Update server_config.template file, set CPU compute only default +- MS-419 - Move index_file_size from IndexParam to TableSchema ## New Feature - MS-343 - Implement ResourceMgr diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index dcf2cd609183bad50b0a986eebc93597e4b955e1..7145db04b2473e8f18482b83da13a00707c0404e 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -55,7 +55,9 @@ DBImpl::DBImpl(const Options& options) } Status DBImpl::CreateTable(meta::TableSchema& table_schema) { - return meta_ptr_->CreateTable(table_schema); + meta::TableSchema temp_schema = table_schema; + temp_schema.index_file_size_ *= ONE_MB; + return meta_ptr_->CreateTable(temp_schema); } Status DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& dates) { diff --git a/cpp/src/db/Types.h b/cpp/src/db/Types.h index 7a276913e3064ef9fc6c048c06054937ba006630..322b22048a7bc3ab178f5b8a75f2ed1d2f520173 100644 --- a/cpp/src/db/Types.h +++ b/cpp/src/db/Types.h @@ -24,7 +24,6 @@ typedef std::vector QueryResults; struct TableIndex { int32_t engine_type_ = (int)EngineType::FAISS_IDMAP; int32_t nlist_ = 16384; - int32_t index_file_size_ = 1024; //MB int32_t metric_type_ = (int)MetricType::L2; }; diff --git a/cpp/src/db/Utils.cpp b/cpp/src/db/Utils.cpp index e227623e231bf0ff986e8d696a75ab907d824fa1..4eec3346b0260b627796f9a191d6479d2d0f5455 100644 --- a/cpp/src/db/Utils.cpp +++ b/cpp/src/db/Utils.cpp @@ -149,7 +149,6 @@ Status DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& bool IsSameIndex(const TableIndex& index1, const TableIndex& index2) { return index1.engine_type_ == index2.engine_type_ && index1.nlist_ == index2.nlist_ - && index1.index_file_size_ == index2.index_file_size_ && index1.metric_type_ == index2.metric_type_; } diff --git a/cpp/src/db/meta/MetaTypes.h b/cpp/src/db/meta/MetaTypes.h index 61af7927469211d964596fab62c63625cd77dcc2..0727528fe935f5401c2acce79a71ad9736869e8a 100644 --- a/cpp/src/db/meta/MetaTypes.h +++ b/cpp/src/db/meta/MetaTypes.h @@ -40,9 +40,9 @@ struct TableSchema { uint16_t dimension_ = 0; int64_t created_on_ = 0; int64_t flag_ = 0; + int32_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE; int32_t engine_type_ = DEFAULT_ENGINE_TYPE; int32_t nlist_ = DEFAULT_NLIST; - int32_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE; int32_t metric_type_ = DEFAULT_METRIC_TYPE; }; // TableSchema @@ -69,9 +69,9 @@ struct TableFileSchema { std::string location_; int64_t updated_time_ = 0; int64_t created_on_ = 0; + int64_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE; //not persist to meta int32_t engine_type_ = DEFAULT_ENGINE_TYPE; int32_t nlist_ = DEFAULT_NLIST; //not persist to meta - int32_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE; //not persist to meta int32_t metric_type_ = DEFAULT_METRIC_TYPE; //not persist to meta }; // TableFileSchema diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index 459da69deb0abb57491132e810d59cec1f11b649..92af8e2a677d9697c7dd9a03302e0908c25d17e4 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -138,9 +138,9 @@ Status MySQLMetaImpl::Initialize() { "dimension SMALLINT NOT NULL, " << "created_on BIGINT NOT NULL, " << "flag BIGINT DEFAULT 0 NOT NULL, " << + "index_file_size INT DEFAULT 1024 NOT NULL, " << "engine_type INT DEFAULT 1 NOT NULL, " << "nlist INT DEFAULT 16384 NOT NULL, " << - "index_file_size INT DEFAULT 1024 NOT NULL, " << "metric_type INT DEFAULT 1 NOT NULL);"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); @@ -407,7 +407,6 @@ Status MySQLMetaImpl::UpdateTableIndexParam(const std::string &table_id, const T "created_on = " << created_on << ", " << "engine_type_ = " << index.engine_type_ << ", " << "nlist = " << index.nlist_ << ", " << - "index_file_size = " << index.index_file_size_*ONE_MB << ", " << "metric_type = " << index.metric_type_ << " " << "WHERE id = " << quote << table_id << ";"; @@ -504,7 +503,6 @@ Status MySQLMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex index.engine_type_ = resRow["engine_type"]; index.nlist_ = resRow["nlist"]; - index.index_file_size_ = resRow["index_file_size"]/ONE_MB; index.metric_type_ = resRow["metric_type"]; } else { return Status::NotFound("Table " + table_id + " not found"); @@ -694,12 +692,12 @@ Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { table_schema.dimension_ = resRow["dimension"]; + table_schema.index_file_size_ = resRow["index_file_size"]; + table_schema.engine_type_ = resRow["engine_type"]; table_schema.nlist_ = resRow["nlist"]; - table_schema.index_file_size_ = resRow["index_file_size"]; - table_schema.metric_type_ = resRow["metric_type"]; } else { return Status::NotFound("Table " + table_schema.table_id_ + " not found"); @@ -790,12 +788,12 @@ Status MySQLMetaImpl::AllTables(std::vector &table_schema_array) { table_schema.dimension_ = resRow["dimension"]; + table_schema.index_file_size_ = resRow["index_file_size"]; + table_schema.engine_type_ = resRow["engine_type"]; table_schema.nlist_ = resRow["nlist"]; - table_schema.index_file_size_ = resRow["index_file_size"]; - table_schema.metric_type_ = resRow["metric_type"]; table_schema_array.emplace_back(table_schema); @@ -833,9 +831,9 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { file_schema.row_count_ = 0; file_schema.created_on_ = utils::GetMicroSecTimeStamp(); file_schema.updated_time_ = file_schema.created_on_; + file_schema.index_file_size_ = table_schema.index_file_size_; file_schema.engine_type_ = table_schema.engine_type_; file_schema.nlist_ = table_schema.nlist_; - file_schema.index_file_size_ = table_schema.index_file_size_; file_schema.metric_type_ = table_schema.metric_type_; utils::GetTableFilePath(options_, file_schema); @@ -950,10 +948,10 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { groups[table_file.table_id_] = table_schema; } - table_file.nlist_ = groups[table_file.table_id_].nlist_; + table_file.dimension_ = groups[table_file.table_id_].dimension_; table_file.index_file_size_ = groups[table_file.table_id_].index_file_size_; + table_file.nlist_ = groups[table_file.table_id_].nlist_; table_file.metric_type_ = groups[table_file.table_id_].metric_type_; - table_file.dimension_ = groups[table_file.table_id_].dimension_; utils::GetTableFilePath(options_, table_file); @@ -1043,12 +1041,12 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, resRow["table_id"].to_string(table_id_str); table_file.table_id_ = table_id_str; + table_file.index_file_size_ = table_schema.index_file_size_; + table_file.engine_type_ = resRow["engine_type"]; table_file.nlist_ = table_schema.nlist_; - table_file.index_file_size_ = table_schema.index_file_size_; - table_file.metric_type_ = table_schema.metric_type_; std::string file_id; @@ -1157,12 +1155,12 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, resRow["table_id"].to_string(table_id_str); table_file.table_id_ = table_id_str; + table_file.index_file_size_ = table_schema.index_file_size_; + table_file.engine_type_ = resRow["engine_type"]; table_file.nlist_ = table_schema.nlist_; - table_file.index_file_size_ = table_schema.index_file_size_; - table_file.metric_type_ = table_schema.metric_type_; std::string file_id; @@ -1259,12 +1257,12 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, table_file.date_ = resRow["date"]; + table_file.index_file_size_ = table_schema.index_file_size_; + table_file.engine_type_ = resRow["engine_type"]; table_file.nlist_ = table_schema.nlist_; - table_file.index_file_size_ = table_schema.index_file_size_; - table_file.metric_type_ = table_schema.metric_type_; table_file.created_on_ = resRow["created_on"]; @@ -1344,12 +1342,12 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id, file_schema.table_id_ = table_id; + file_schema.index_file_size_ = table_schema.index_file_size_; + file_schema.engine_type_ = resRow["engine_type"]; file_schema.nlist_ = table_schema.nlist_; - file_schema.index_file_size_ = table_schema.index_file_size_; - file_schema.metric_type_ = table_schema.metric_type_; std::string file_id; diff --git a/cpp/src/db/meta/SqliteMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp index 99f0b104a9c20bd3355948ba6af8d19f67d01c4b..d0d64236882566960c841b5de7630857c9bc5dc4 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -45,9 +45,9 @@ inline auto StoragePrototype(const std::string &path) { make_column("dimension", &TableSchema::dimension_), make_column("created_on", &TableSchema::created_on_), make_column("flag", &TableSchema::flag_, default_value(0)), + make_column("index_file_size", &TableSchema::index_file_size_), make_column("engine_type", &TableSchema::engine_type_), make_column("nlist", &TableSchema::nlist_), - make_column("index_file_size", &TableSchema::index_file_size_), make_column("metric_type", &TableSchema::metric_type_)), make_table("TableFiles", make_column("id", &TableFileSchema::id_, primary_key()), @@ -251,9 +251,9 @@ Status SqliteMetaImpl::DescribeTable(TableSchema &table_schema) { &TableSchema::dimension_, &TableSchema::created_on_, &TableSchema::flag_, + &TableSchema::index_file_size_, &TableSchema::engine_type_, &TableSchema::nlist_, - &TableSchema::index_file_size_, &TableSchema::metric_type_), where(c(&TableSchema::table_id_) == table_schema.table_id_ and c(&TableSchema::state_) != (int)TableSchema::TO_DELETE)); @@ -264,9 +264,9 @@ Status SqliteMetaImpl::DescribeTable(TableSchema &table_schema) { table_schema.dimension_ = std::get<2>(groups[0]); table_schema.created_on_ = std::get<3>(groups[0]); table_schema.flag_ = std::get<4>(groups[0]); - table_schema.engine_type_ = std::get<5>(groups[0]); - table_schema.nlist_ = std::get<6>(groups[0]); - table_schema.index_file_size_ = std::get<7>(groups[0]); + table_schema.index_file_size_ = std::get<5>(groups[0]); + table_schema.engine_type_ = std::get<6>(groups[0]); + table_schema.nlist_ = std::get<7>(groups[0]); table_schema.metric_type_ = std::get<8>(groups[0]); } else { return Status::NotFound("Table " + table_schema.table_id_ + " not found"); @@ -344,7 +344,8 @@ Status SqliteMetaImpl::UpdateTableIndexParam(const std::string &table_id, const &TableSchema::state_, &TableSchema::dimension_, &TableSchema::created_on_, - &TableSchema::flag_), + &TableSchema::flag_, + &TableSchema::index_file_size_), where(c(&TableSchema::table_id_) == table_id and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); @@ -356,9 +357,9 @@ Status SqliteMetaImpl::UpdateTableIndexParam(const std::string &table_id, const table_schema.dimension_ = std::get<2>(tables[0]); table_schema.created_on_ = std::get<3>(tables[0]); table_schema.flag_ = std::get<4>(tables[0]); + table_schema.index_file_size_ = std::get<5>(tables[0]); table_schema.engine_type_ = index.engine_type_; table_schema.nlist_ = index.nlist_; - table_schema.index_file_size_ = index.index_file_size_*ONE_MB; table_schema.metric_type_ = index.metric_type_; ConnectorPtr->update(table_schema); @@ -420,8 +421,7 @@ Status SqliteMetaImpl::DescribeTableIndex(const std::string &table_id, TableInde if (groups.size() == 1) { index.engine_type_ = std::get<0>(groups[0]); index.nlist_ = std::get<1>(groups[0]); - index.index_file_size_ = std::get<2>(groups[0])/ONE_MB; - index.metric_type_ = std::get<3>(groups[0]); + index.metric_type_ = std::get<2>(groups[0]); } else { return Status::NotFound("Table " + table_id + " not found"); } @@ -499,9 +499,9 @@ Status SqliteMetaImpl::AllTables(std::vector& table_schema_array) { &TableSchema::dimension_, &TableSchema::created_on_, &TableSchema::flag_, + &TableSchema::index_file_size_, &TableSchema::engine_type_, &TableSchema::nlist_, - &TableSchema::index_file_size_, &TableSchema::metric_type_), where(c(&TableSchema::state_) != (int)TableSchema::TO_DELETE)); for (auto &table : selected) { @@ -511,9 +511,9 @@ Status SqliteMetaImpl::AllTables(std::vector& table_schema_array) { schema.dimension_ = std::get<2>(table); schema.created_on_ = std::get<3>(table); schema.flag_ = std::get<4>(table); - schema.engine_type_ = std::get<5>(table); - schema.nlist_ = std::get<6>(table); - schema.index_file_size_ = std::get<7>(table); + schema.index_file_size_ = std::get<5>(table); + schema.engine_type_ = std::get<6>(table); + schema.nlist_ = std::get<7>(table); schema.metric_type_ = std::get<8>(table); table_schema_array.emplace_back(schema); @@ -546,9 +546,9 @@ Status SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) { file_schema.row_count_ = 0; file_schema.created_on_ = utils::GetMicroSecTimeStamp(); file_schema.updated_time_ = file_schema.created_on_; + file_schema.index_file_size_ = table_schema.index_file_size_; file_schema.engine_type_ = table_schema.engine_type_; file_schema.nlist_ = table_schema.nlist_; - file_schema.index_file_size_ = table_schema.index_file_size_; file_schema.metric_type_ = table_schema.metric_type_; //multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here @@ -609,10 +609,10 @@ Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) { } groups[table_file.table_id_] = table_schema; } - table_file.nlist_ = groups[table_file.table_id_].nlist_; + table_file.dimension_ = groups[table_file.table_id_].dimension_; table_file.index_file_size_ = groups[table_file.table_id_].index_file_size_; + table_file.nlist_ = groups[table_file.table_id_].nlist_; table_file.metric_type_ = groups[table_file.table_id_].metric_type_; - table_file.dimension_ = groups[table_file.table_id_].dimension_; files.push_back(table_file); } @@ -662,9 +662,11 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, table_file.row_count_ = std::get<5>(file); table_file.date_ = std::get<6>(file); table_file.engine_type_ = std::get<7>(file); - table_file.metric_type_ = table_schema.metric_type_; - table_file.nlist_ = table_schema.nlist_; table_file.dimension_ = table_schema.dimension_; + table_file.index_file_size_ = table_schema.index_file_size_; + table_file.nlist_ = table_schema.nlist_; + table_file.metric_type_ = table_schema.metric_type_; + utils::GetTableFilePath(options_, table_file); auto dateItr = files.find(table_file.date_); if (dateItr == files.end()) { @@ -705,10 +707,10 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, table_file.row_count_ = std::get<5>(file); table_file.date_ = std::get<6>(file); table_file.engine_type_ = std::get<7>(file); - table_file.nlist_ = table_schema.nlist_; + table_file.dimension_ = table_schema.dimension_; table_file.index_file_size_ = table_schema.index_file_size_; + table_file.nlist_ = table_schema.nlist_; table_file.metric_type_ = table_schema.metric_type_; - table_file.dimension_ = table_schema.dimension_; utils::GetTableFilePath(options_, table_file); auto dateItr = files.find(table_file.date_); @@ -786,8 +788,8 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, table_file.date_ = std::get<6>(file); table_file.engine_type_ = std::get<7>(file); table_file.dimension_ = table_schema.dimension_; - table_file.nlist_ = table_schema.nlist_; table_file.index_file_size_ = table_schema.index_file_size_; + table_file.nlist_ = table_schema.nlist_; table_file.metric_type_ = table_schema.metric_type_; utils::GetTableFilePath(options_, table_file); @@ -848,8 +850,8 @@ Status SqliteMetaImpl::FilesToMerge(const std::string &table_id, table_file.date_ = std::get<6>(file); table_file.created_on_ = std::get<7>(file); table_file.dimension_ = table_schema.dimension_; - table_file.nlist_ = table_schema.nlist_; table_file.index_file_size_ = table_schema.index_file_size_; + table_file.nlist_ = table_schema.nlist_; table_file.metric_type_ = table_schema.metric_type_; utils::GetTableFilePath(options_, table_file); @@ -900,11 +902,11 @@ Status SqliteMetaImpl::GetTableFiles(const std::string& table_id, file_schema.row_count_ = std::get<4>(file); file_schema.date_ = std::get<5>(file); file_schema.engine_type_ = std::get<6>(file); - file_schema.nlist_ = table_schema.nlist_; - file_schema.index_file_size_ = table_schema.index_file_size_; - file_schema.metric_type_ = table_schema.metric_type_; file_schema.created_on_ = std::get<7>(file); file_schema.dimension_ = table_schema.dimension_; + file_schema.index_file_size_ = table_schema.index_file_size_; + file_schema.nlist_ = table_schema.nlist_; + file_schema.metric_type_ = table_schema.metric_type_; utils::GetTableFilePath(options_, file_schema); diff --git a/cpp/src/grpc/gen-milvus/milvus.pb.cc b/cpp/src/grpc/gen-milvus/milvus.pb.cc index 25f115db18dec48bcad8d1d378e39db6cbbef007..7cb053a9bb35b0d09c8eae54502cc430d8ae45b1 100644 --- a/cpp/src/grpc/gen-milvus/milvus.pb.cc +++ b/cpp/src/grpc/gen-milvus/milvus.pb.cc @@ -366,6 +366,7 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT ~0u, // no _weak_field_map_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, table_name_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, dimension_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, index_file_size_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::Range, _internal_metadata_), ~0u, // no _extensions_ @@ -459,7 +460,6 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT ~0u, // no _weak_field_map_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, index_type_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, nlist_), - PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, index_file_size_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, metric_type_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::IndexParam, _internal_metadata_), @@ -479,19 +479,19 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = { { 0, -1, sizeof(::milvus::grpc::TableName)}, { 7, -1, sizeof(::milvus::grpc::TableSchema)}, - { 14, -1, sizeof(::milvus::grpc::Range)}, - { 21, -1, sizeof(::milvus::grpc::RowRecord)}, - { 27, -1, sizeof(::milvus::grpc::InsertParam)}, - { 35, -1, sizeof(::milvus::grpc::VectorIds)}, - { 42, -1, sizeof(::milvus::grpc::SearchParam)}, - { 52, -1, sizeof(::milvus::grpc::SearchInFilesParam)}, - { 59, -1, sizeof(::milvus::grpc::QueryResult)}, - { 66, -1, sizeof(::milvus::grpc::TopKQueryResult)}, - { 73, -1, sizeof(::milvus::grpc::StringReply)}, - { 80, -1, sizeof(::milvus::grpc::BoolReply)}, - { 87, -1, sizeof(::milvus::grpc::TableRowCount)}, - { 94, -1, sizeof(::milvus::grpc::Command)}, - { 100, -1, sizeof(::milvus::grpc::Index)}, + { 15, -1, sizeof(::milvus::grpc::Range)}, + { 22, -1, sizeof(::milvus::grpc::RowRecord)}, + { 28, -1, sizeof(::milvus::grpc::InsertParam)}, + { 36, -1, sizeof(::milvus::grpc::VectorIds)}, + { 43, -1, sizeof(::milvus::grpc::SearchParam)}, + { 53, -1, sizeof(::milvus::grpc::SearchInFilesParam)}, + { 60, -1, sizeof(::milvus::grpc::QueryResult)}, + { 67, -1, sizeof(::milvus::grpc::TopKQueryResult)}, + { 74, -1, sizeof(::milvus::grpc::StringReply)}, + { 81, -1, sizeof(::milvus::grpc::BoolReply)}, + { 88, -1, sizeof(::milvus::grpc::TableRowCount)}, + { 95, -1, sizeof(::milvus::grpc::Command)}, + { 101, -1, sizeof(::milvus::grpc::Index)}, { 109, -1, sizeof(::milvus::grpc::IndexParam)}, { 116, -1, sizeof(::milvus::grpc::DeleteByRangeParam)}, }; @@ -519,35 +519,35 @@ static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = const char descriptor_table_protodef_milvus_2eproto[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = "\n\014milvus.proto\022\013milvus.grpc\032\014status.prot" "o\"D\n\tTableName\022#\n\006status\030\001 \001(\0132\023.milvus." - "grpc.Status\022\022\n\ntable_name\030\002 \001(\t\"L\n\013Table" + "grpc.Status\022\022\n\ntable_name\030\002 \001(\t\"e\n\013Table" "Schema\022*\n\ntable_name\030\001 \001(\0132\026.milvus.grpc" - ".TableName\022\021\n\tdimension\030\002 \001(\003\"/\n\005Range\022\023" - "\n\013start_value\030\001 \001(\t\022\021\n\tend_value\030\002 \001(\t\" " - "\n\tRowRecord\022\023\n\013vector_data\030\001 \003(\002\"i\n\013Inse" - "rtParam\022\022\n\ntable_name\030\001 \001(\t\0220\n\020row_recor" - "d_array\030\002 \003(\0132\026.milvus.grpc.RowRecord\022\024\n" - "\014row_id_array\030\003 \003(\003\"I\n\tVectorIds\022#\n\006stat" - "us\030\001 \001(\0132\023.milvus.grpc.Status\022\027\n\017vector_" - "id_array\030\002 \003(\003\"\242\001\n\013SearchParam\022\022\n\ntable_" - "name\030\001 \001(\t\0222\n\022query_record_array\030\002 \003(\0132\026" - ".milvus.grpc.RowRecord\022-\n\021query_range_ar" - "ray\030\003 \003(\0132\022.milvus.grpc.Range\022\014\n\004topk\030\004 " - "\001(\003\022\016\n\006nprobe\030\005 \001(\003\"[\n\022SearchInFilesPara" - "m\022\025\n\rfile_id_array\030\001 \003(\t\022.\n\014search_param" - "\030\002 \001(\0132\030.milvus.grpc.SearchParam\"+\n\013Quer" - "yResult\022\n\n\002id\030\001 \001(\003\022\020\n\010distance\030\002 \001(\001\"m\n" - "\017TopKQueryResult\022#\n\006status\030\001 \001(\0132\023.milvu" - "s.grpc.Status\0225\n\023query_result_arrays\030\002 \003" - "(\0132\030.milvus.grpc.QueryResult\"H\n\013StringRe" - "ply\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.Status" - "\022\024\n\014string_reply\030\002 \001(\t\"D\n\tBoolReply\022#\n\006s" - "tatus\030\001 \001(\0132\023.milvus.grpc.Status\022\022\n\nbool" - "_reply\030\002 \001(\010\"M\n\rTableRowCount\022#\n\006status\030" - "\001 \001(\0132\023.milvus.grpc.Status\022\027\n\017table_row_" - "count\030\002 \001(\003\"\026\n\007Command\022\013\n\003cmd\030\001 \001(\t\"X\n\005I" - "ndex\022\022\n\nindex_type\030\001 \001(\005\022\r\n\005nlist\030\002 \001(\005\022" - "\027\n\017index_file_size\030\003 \001(\005\022\023\n\013metric_type\030" - "\004 \001(\005\"[\n\nIndexParam\022*\n\ntable_name\030\001 \001(\0132" + ".TableName\022\021\n\tdimension\030\002 \001(\003\022\027\n\017index_f" + "ile_size\030\003 \001(\003\"/\n\005Range\022\023\n\013start_value\030\001" + " \001(\t\022\021\n\tend_value\030\002 \001(\t\" \n\tRowRecord\022\023\n\013" + "vector_data\030\001 \003(\002\"i\n\013InsertParam\022\022\n\ntabl" + "e_name\030\001 \001(\t\0220\n\020row_record_array\030\002 \003(\0132\026" + ".milvus.grpc.RowRecord\022\024\n\014row_id_array\030\003" + " \003(\003\"I\n\tVectorIds\022#\n\006status\030\001 \001(\0132\023.milv" + "us.grpc.Status\022\027\n\017vector_id_array\030\002 \003(\003\"" + "\242\001\n\013SearchParam\022\022\n\ntable_name\030\001 \001(\t\0222\n\022q" + "uery_record_array\030\002 \003(\0132\026.milvus.grpc.Ro" + "wRecord\022-\n\021query_range_array\030\003 \003(\0132\022.mil" + "vus.grpc.Range\022\014\n\004topk\030\004 \001(\003\022\016\n\006nprobe\030\005" + " \001(\003\"[\n\022SearchInFilesParam\022\025\n\rfile_id_ar" + "ray\030\001 \003(\t\022.\n\014search_param\030\002 \001(\0132\030.milvus" + ".grpc.SearchParam\"+\n\013QueryResult\022\n\n\002id\030\001" + " \001(\003\022\020\n\010distance\030\002 \001(\001\"m\n\017TopKQueryResul" + "t\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.Status\0225" + "\n\023query_result_arrays\030\002 \003(\0132\030.milvus.grp" + "c.QueryResult\"H\n\013StringReply\022#\n\006status\030\001" + " \001(\0132\023.milvus.grpc.Status\022\024\n\014string_repl" + "y\030\002 \001(\t\"D\n\tBoolReply\022#\n\006status\030\001 \001(\0132\023.m" + "ilvus.grpc.Status\022\022\n\nbool_reply\030\002 \001(\010\"M\n" + "\rTableRowCount\022#\n\006status\030\001 \001(\0132\023.milvus." + "grpc.Status\022\027\n\017table_row_count\030\002 \001(\003\"\026\n\007" + "Command\022\013\n\003cmd\030\001 \001(\t\"\?\n\005Index\022\022\n\nindex_t" + "ype\030\001 \001(\005\022\r\n\005nlist\030\002 \001(\005\022\023\n\013metric_type\030" + "\003 \001(\005\"[\n\nIndexParam\022*\n\ntable_name\030\001 \001(\0132" "\026.milvus.grpc.TableName\022!\n\005index\030\002 \001(\0132\022" ".milvus.grpc.Index\"K\n\022DeleteByRangeParam" "\022!\n\005range\030\001 \001(\0132\022.milvus.grpc.Range\022\022\n\nt" @@ -978,15 +978,17 @@ TableSchema::TableSchema(const TableSchema& from) } else { table_name_ = nullptr; } - dimension_ = from.dimension_; + ::memcpy(&dimension_, &from.dimension_, + static_cast(reinterpret_cast(&index_file_size_) - + reinterpret_cast(&dimension_)) + sizeof(index_file_size_)); // @@protoc_insertion_point(copy_constructor:milvus.grpc.TableSchema) } void TableSchema::SharedCtor() { ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_TableSchema_milvus_2eproto.base); ::memset(&table_name_, 0, static_cast( - reinterpret_cast(&dimension_) - - reinterpret_cast(&table_name_)) + sizeof(dimension_)); + reinterpret_cast(&index_file_size_) - + reinterpret_cast(&table_name_)) + sizeof(index_file_size_)); } TableSchema::~TableSchema() { @@ -1017,7 +1019,9 @@ void TableSchema::Clear() { delete table_name_; } table_name_ = nullptr; - dimension_ = PROTOBUF_LONGLONG(0); + ::memset(&dimension_, 0, static_cast( + reinterpret_cast(&index_file_size_) - + reinterpret_cast(&dimension_)) + sizeof(index_file_size_)); _internal_metadata_.Clear(); } @@ -1043,6 +1047,13 @@ const char* TableSchema::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID CHK_(ptr); } else goto handle_unusual; continue; + // int64 index_file_size = 3; + case 3: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) { + index_file_size_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; default: { handle_unusual: if ((tag & 7) == 4 || tag == 0) { @@ -1097,6 +1108,19 @@ bool TableSchema::MergePartialFromCodedStream( break; } + // int64 index_file_size = 3; + case 3: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (24 & 0xFF)) { + + DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< + ::PROTOBUF_NAMESPACE_ID::int64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT64>( + input, &index_file_size_))); + } else { + goto handle_unusual; + } + break; + } + default: { handle_unusual: if (tag == 0) { @@ -1135,6 +1159,11 @@ void TableSchema::SerializeWithCachedSizes( ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(2, this->dimension(), output); } + // int64 index_file_size = 3; + if (this->index_file_size() != 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(3, this->index_file_size(), output); + } + if (_internal_metadata_.have_unknown_fields()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields( _internal_metadata_.unknown_fields(), output); @@ -1160,6 +1189,11 @@ void TableSchema::SerializeWithCachedSizes( target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(2, this->dimension(), target); } + // int64 index_file_size = 3; + if (this->index_file_size() != 0) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(3, this->index_file_size(), target); + } + if (_internal_metadata_.have_unknown_fields()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields(), target); @@ -1195,6 +1229,13 @@ size_t TableSchema::ByteSizeLong() const { this->dimension()); } + // int64 index_file_size = 3; + if (this->index_file_size() != 0) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int64Size( + this->index_file_size()); + } + int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); SetCachedSize(cached_size); return total_size; @@ -1228,6 +1269,9 @@ void TableSchema::MergeFrom(const TableSchema& from) { if (from.dimension() != 0) { set_dimension(from.dimension()); } + if (from.index_file_size() != 0) { + set_index_file_size(from.index_file_size()); + } } void TableSchema::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { @@ -1253,6 +1297,7 @@ void TableSchema::InternalSwap(TableSchema* other) { _internal_metadata_.Swap(&other->_internal_metadata_); swap(table_name_, other->table_name_); swap(dimension_, other->dimension_); + swap(index_file_size_, other->index_file_size_); } ::PROTOBUF_NAMESPACE_ID::Metadata TableSchema::GetMetadata() const { @@ -5331,16 +5376,9 @@ const char* Index::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::inte CHK_(ptr); } else goto handle_unusual; continue; - // int32 index_file_size = 3; + // int32 metric_type = 3; case 3: if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) { - index_file_size_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); - CHK_(ptr); - } else goto handle_unusual; - continue; - // int32 metric_type = 4; - case 4: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 32)) { metric_type_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); CHK_(ptr); } else goto handle_unusual; @@ -5401,23 +5439,10 @@ bool Index::MergePartialFromCodedStream( break; } - // int32 index_file_size = 3; + // int32 metric_type = 3; case 3: { if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (24 & 0xFF)) { - DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< - ::PROTOBUF_NAMESPACE_ID::int32, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT32>( - input, &index_file_size_))); - } else { - goto handle_unusual; - } - break; - } - - // int32 metric_type = 4; - case 4: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (32 & 0xFF)) { - DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< ::PROTOBUF_NAMESPACE_ID::int32, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT32>( input, &metric_type_))); @@ -5464,14 +5489,9 @@ void Index::SerializeWithCachedSizes( ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(2, this->nlist(), output); } - // int32 index_file_size = 3; - if (this->index_file_size() != 0) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(3, this->index_file_size(), output); - } - - // int32 metric_type = 4; + // int32 metric_type = 3; if (this->metric_type() != 0) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(4, this->metric_type(), output); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(3, this->metric_type(), output); } if (_internal_metadata_.have_unknown_fields()) { @@ -5497,14 +5517,9 @@ void Index::SerializeWithCachedSizes( target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(2, this->nlist(), target); } - // int32 index_file_size = 3; - if (this->index_file_size() != 0) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(3, this->index_file_size(), target); - } - - // int32 metric_type = 4; + // int32 metric_type = 3; if (this->metric_type() != 0) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(4, this->metric_type(), target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(3, this->metric_type(), target); } if (_internal_metadata_.have_unknown_fields()) { @@ -5542,14 +5557,7 @@ size_t Index::ByteSizeLong() const { this->nlist()); } - // int32 index_file_size = 3; - if (this->index_file_size() != 0) { - total_size += 1 + - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( - this->index_file_size()); - } - - // int32 metric_type = 4; + // int32 metric_type = 3; if (this->metric_type() != 0) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( @@ -5589,9 +5597,6 @@ void Index::MergeFrom(const Index& from) { if (from.nlist() != 0) { set_nlist(from.nlist()); } - if (from.index_file_size() != 0) { - set_index_file_size(from.index_file_size()); - } if (from.metric_type() != 0) { set_metric_type(from.metric_type()); } @@ -5620,7 +5625,6 @@ void Index::InternalSwap(Index* other) { _internal_metadata_.Swap(&other->_internal_metadata_); swap(index_type_, other->index_type_); swap(nlist_, other->nlist_); - swap(index_file_size_, other->index_file_size_); swap(metric_type_, other->metric_type_); } diff --git a/cpp/src/grpc/gen-milvus/milvus.pb.h b/cpp/src/grpc/gen-milvus/milvus.pb.h index 2c26bc4e4c7531b09032dcebe20bb0d38ffe400b..374069d35fc226ea5633bb609f3f3732ffa9e276 100644 --- a/cpp/src/grpc/gen-milvus/milvus.pb.h +++ b/cpp/src/grpc/gen-milvus/milvus.pb.h @@ -396,6 +396,7 @@ class TableSchema : enum : int { kTableNameFieldNumber = 1, kDimensionFieldNumber = 2, + kIndexFileSizeFieldNumber = 3, }; // .milvus.grpc.TableName table_name = 1; bool has_table_name() const; @@ -410,6 +411,11 @@ class TableSchema : ::PROTOBUF_NAMESPACE_ID::int64 dimension() const; void set_dimension(::PROTOBUF_NAMESPACE_ID::int64 value); + // int64 index_file_size = 3; + void clear_index_file_size(); + ::PROTOBUF_NAMESPACE_ID::int64 index_file_size() const; + void set_index_file_size(::PROTOBUF_NAMESPACE_ID::int64 value); + // @@protoc_insertion_point(class_scope:milvus.grpc.TableSchema) private: class _Internal; @@ -417,6 +423,7 @@ class TableSchema : ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_; ::milvus::grpc::TableName* table_name_; ::PROTOBUF_NAMESPACE_ID::int64 dimension_; + ::PROTOBUF_NAMESPACE_ID::int64 index_file_size_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; friend struct ::TableStruct_milvus_2eproto; }; @@ -2318,8 +2325,7 @@ class Index : enum : int { kIndexTypeFieldNumber = 1, kNlistFieldNumber = 2, - kIndexFileSizeFieldNumber = 3, - kMetricTypeFieldNumber = 4, + kMetricTypeFieldNumber = 3, }; // int32 index_type = 1; void clear_index_type(); @@ -2331,12 +2337,7 @@ class Index : ::PROTOBUF_NAMESPACE_ID::int32 nlist() const; void set_nlist(::PROTOBUF_NAMESPACE_ID::int32 value); - // int32 index_file_size = 3; - void clear_index_file_size(); - ::PROTOBUF_NAMESPACE_ID::int32 index_file_size() const; - void set_index_file_size(::PROTOBUF_NAMESPACE_ID::int32 value); - - // int32 metric_type = 4; + // int32 metric_type = 3; void clear_metric_type(); ::PROTOBUF_NAMESPACE_ID::int32 metric_type() const; void set_metric_type(::PROTOBUF_NAMESPACE_ID::int32 value); @@ -2348,7 +2349,6 @@ class Index : ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_; ::PROTOBUF_NAMESPACE_ID::int32 index_type_; ::PROTOBUF_NAMESPACE_ID::int32 nlist_; - ::PROTOBUF_NAMESPACE_ID::int32 index_file_size_; ::PROTOBUF_NAMESPACE_ID::int32 metric_type_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; friend struct ::TableStruct_milvus_2eproto; @@ -2820,6 +2820,20 @@ inline void TableSchema::set_dimension(::PROTOBUF_NAMESPACE_ID::int64 value) { // @@protoc_insertion_point(field_set:milvus.grpc.TableSchema.dimension) } +// int64 index_file_size = 3; +inline void TableSchema::clear_index_file_size() { + index_file_size_ = PROTOBUF_LONGLONG(0); +} +inline ::PROTOBUF_NAMESPACE_ID::int64 TableSchema::index_file_size() const { + // @@protoc_insertion_point(field_get:milvus.grpc.TableSchema.index_file_size) + return index_file_size_; +} +inline void TableSchema::set_index_file_size(::PROTOBUF_NAMESPACE_ID::int64 value) { + + index_file_size_ = value; + // @@protoc_insertion_point(field_set:milvus.grpc.TableSchema.index_file_size) +} + // ------------------------------------------------------------------- // Range @@ -3841,21 +3855,7 @@ inline void Index::set_nlist(::PROTOBUF_NAMESPACE_ID::int32 value) { // @@protoc_insertion_point(field_set:milvus.grpc.Index.nlist) } -// int32 index_file_size = 3; -inline void Index::clear_index_file_size() { - index_file_size_ = 0; -} -inline ::PROTOBUF_NAMESPACE_ID::int32 Index::index_file_size() const { - // @@protoc_insertion_point(field_get:milvus.grpc.Index.index_file_size) - return index_file_size_; -} -inline void Index::set_index_file_size(::PROTOBUF_NAMESPACE_ID::int32 value) { - - index_file_size_ = value; - // @@protoc_insertion_point(field_set:milvus.grpc.Index.index_file_size) -} - -// int32 metric_type = 4; +// int32 metric_type = 3; inline void Index::clear_metric_type() { metric_type_ = 0; } diff --git a/cpp/src/grpc/milvus.proto b/cpp/src/grpc/milvus.proto index c20eab6e56460dd3c3d0de2ecc012ec5ea91780c..ff95a305ab0ecd0aeb89b38140f2465afe145619 100644 --- a/cpp/src/grpc/milvus.proto +++ b/cpp/src/grpc/milvus.proto @@ -18,6 +18,7 @@ message TableName { message TableSchema { TableName table_name = 1; int64 dimension = 2; + int64 index_file_size = 3; } /** @@ -126,8 +127,7 @@ message Command { message Index { int32 index_type = 1; int32 nlist = 2; - int32 index_file_size = 3; - int32 metric_type = 4; + int32 metric_type = 3; } /** diff --git a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp index 3f43a9619c5b0d5d2162de64d612194ab516de83..87313d22944c865aa14320d73538cc2825746d5d 100644 --- a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp @@ -22,6 +22,7 @@ namespace { const std::string TABLE_NAME = GetTableName(); constexpr int64_t TABLE_DIMENSION = 512; + constexpr int64_t TABLE_INDEX_FILE_SIZE = 768; constexpr int64_t BATCH_ROW_COUNT = 100000; constexpr int64_t NQ = 10; constexpr int64_t TOP_K = 10; @@ -94,6 +95,7 @@ namespace { TableSchema tb_schema; tb_schema.table_name = TABLE_NAME; tb_schema.dimension = TABLE_DIMENSION; + tb_schema.index_file_size = TABLE_INDEX_FILE_SIZE; return tb_schema; } @@ -286,7 +288,6 @@ ClientTest::Test(const std::string& address, const std::string& port) { index.table_name = TABLE_NAME; index.index_type = IndexType::gpu_ivfflat; index.nlist = 1000; - index.index_file_size = 1024; index.metric_type = 1; Status stat = conn->CreateIndex(index); std::cout << "CreateIndex function call status: " << stat.ToString() << std::endl; diff --git a/cpp/src/sdk/grpc/ClientProxy.cpp b/cpp/src/sdk/grpc/ClientProxy.cpp index 6421a34d198af568e56043ffe827ddf9db24f60f..a135910383315402377360ee9c4456ed510a39e5 100644 --- a/cpp/src/sdk/grpc/ClientProxy.cpp +++ b/cpp/src/sdk/grpc/ClientProxy.cpp @@ -83,6 +83,7 @@ ClientProxy::CreateTable(const TableSchema ¶m) { ::milvus::grpc::TableSchema schema; schema.mutable_table_name()->set_table_name(param.table_name); schema.set_dimension(param.dimension); + schema.set_index_file_size(param.index_file_size); return client_ptr_->CreateTable(schema); } catch (std::exception &ex) { @@ -119,7 +120,6 @@ ClientProxy::CreateIndex(const IndexParam &index_param) { index_param.table_name); grpc_index_param.mutable_index()->set_index_type((int32_t)index_param.index_type); grpc_index_param.mutable_index()->set_nlist(index_param.nlist); - grpc_index_param.mutable_index()->set_index_file_size(index_param.index_file_size); grpc_index_param.mutable_index()->set_metric_type(index_param.metric_type); return client_ptr_->CreateIndex(grpc_index_param); @@ -272,6 +272,7 @@ ClientProxy::DescribeTable(const std::string &table_name, TableSchema &table_sch table_schema.table_name = grpc_schema.table_name().table_name(); table_schema.dimension = grpc_schema.dimension(); + table_schema.index_file_size = grpc_schema.index_file_size(); return status; } catch (std::exception &ex) { @@ -362,7 +363,6 @@ ClientProxy::DescribeIndex(const std::string &table_name, IndexParam &index_para Status status = client_ptr_->DescribeIndex(grpc_table_name, grpc_index_param); index_param.index_type = (IndexType)(grpc_index_param.mutable_index()->index_type()); index_param.nlist = grpc_index_param.mutable_index()->nlist(); - index_param.index_file_size = grpc_index_param.mutable_index()->index_file_size(); index_param.metric_type = grpc_index_param.mutable_index()->metric_type(); return status; diff --git a/cpp/src/sdk/include/MilvusApi.h b/cpp/src/sdk/include/MilvusApi.h index 766bcbad1d04b3021cd4ec15acbdf937fb332e00..2520d80c647697cae50391ccc983c6e0cc710d20 100644 --- a/cpp/src/sdk/include/MilvusApi.h +++ b/cpp/src/sdk/include/MilvusApi.h @@ -35,9 +35,8 @@ struct ConnectParam { */ struct TableSchema { std::string table_name; ///< Table name - IndexType index_type = IndexType::invalid; ///< Index type int64_t dimension = 0; ///< Vector dimension, must be a positive value - bool store_raw_vector = false; ///< Is vector raw data stored in the table + int64_t index_file_size = 0; ///< Index file size, must be a positive value }; /** @@ -78,7 +77,6 @@ struct IndexParam { std::string table_name; IndexType index_type; int32_t nlist; - int32_t index_file_size; int32_t metric_type; }; diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp index d75ad8163c3a1557f49e53bd4d3c23934595bb42..a185272f269f2048f20b38326dcfbefad9eefefa 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp @@ -134,10 +134,16 @@ CreateTableTask::OnExecute() { return SetError(res, "Invalid table dimension: " + std::to_string(schema_.dimension())); } + res = ValidationUtil::ValidateTableIndexFileSize(schema_.index_file_size()); + if(res != SERVER_SUCCESS) { + return SetError(res, "Invalid index file size: " + std::to_string(schema_.index_file_size())); + } + //step 2: construct table schema engine::meta::TableSchema table_info; - table_info.dimension_ = (uint16_t) schema_.dimension(); table_info.table_id_ = schema_.table_name().table_name(); + table_info.dimension_ = (uint16_t) schema_.dimension(); + table_info.index_file_size_ = schema_.index_file_size(); //step 3: create table engine::Status stat = DBWrapper::DB()->CreateTable(table_info); @@ -246,16 +252,10 @@ CreateIndexTask::OnExecute() { return SetError(res, "Invalid index metric type: " + std::to_string(index_param_.mutable_index()->metric_type())); } - res = ValidationUtil::ValidateTableIndexFileSize(index_param_.mutable_index()->index_file_size()); - if(res != SERVER_SUCCESS) { - return SetError(res, "Invalid index file size: " + std::to_string(index_param_.mutable_index()->index_file_size())); - } - //step 2: check table existence engine::TableIndex index; index.engine_type_ = index_param_.mutable_index()->index_type(); index.nlist_ = index_param_.mutable_index()->nlist(); - index.index_file_size_ = index_param_.mutable_index()->index_file_size(); index.metric_type_ = index_param_.mutable_index()->metric_type(); stat = DBWrapper::DB()->CreateIndex(table_name_, index); if (!stat.ok()) { @@ -904,7 +904,6 @@ DescribeIndexTask::OnExecute() { index_param_.mutable_table_name()->set_table_name(table_name_); index_param_.mutable_index()->set_index_type(index.engine_type_); index_param_.mutable_index()->set_nlist(index.nlist_); - index_param_.mutable_index()->set_index_file_size(index.index_file_size_); index_param_.mutable_index()->set_metric_type(index.metric_type_); rc.ElapseFromBegin("totally cost"); diff --git a/cpp/src/utils/ValidationUtil.cpp b/cpp/src/utils/ValidationUtil.cpp index 1ac997f1ab4706b01240596ad5db029d63217c08..0e2e1c2b4f0dfffeb88cb94ac7704930b14859ac 100644 --- a/cpp/src/utils/ValidationUtil.cpp +++ b/cpp/src/utils/ValidationUtil.cpp @@ -76,7 +76,7 @@ ValidationUtil::ValidateTableIndexNlist(int32_t nlist) { } ServerError -ValidationUtil::ValidateTableIndexFileSize(int32_t index_file_size) { +ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) { if(index_file_size <= 0 || index_file_size > index_file_size_limit) { return SERVER_INVALID_INDEX_FILE_SIZE; } diff --git a/cpp/src/utils/ValidationUtil.h b/cpp/src/utils/ValidationUtil.h index 2c90d99dd4d1bfdb5e6304f92592a3177f8636a7..4792500f6719089e49b447f65b1b84f95f1d4a6a 100644 --- a/cpp/src/utils/ValidationUtil.h +++ b/cpp/src/utils/ValidationUtil.h @@ -21,7 +21,7 @@ public: ValidateTableIndexNlist(int32_t nlist); static ServerError - ValidateTableIndexFileSize(int32_t index_file_size); + ValidateTableIndexFileSize(int64_t index_file_size); static ServerError ValidateTableIndexMetricType(int32_t metric_type);