Change to 0.8.0 and rename from preload_table to preload_collection (#1904)

* Create new branch 0.8.0 and change preload_table to preload_collection
Signed-off-by: Njinhai <hai.jin@zilliz.com>

* Fix format
Signed-off-by: NJinHai-CN <hai.jin@zilliz.com>

* Update CHANGELOG
Signed-off-by: Njinhai <hai.jin@zilliz.com>

* Update CHANGELOG
Signed-off-by: Njinhai <hai.jin@zilliz.com>

* update helm version
Signed-off-by: Nzw <zw@zilliz.com>
Co-authored-by: Nzw <zw@zilliz.com>

CHANGELOG.md

@@ -659,7 +659,7 @@ Please mark all change in change log and use the issue from GitHub
 -   MS-65 Implement GetTableRowCount interface
 -   MS-45 Implement DeleteTable interface
 -   MS-75 cmake: change faiss version to 1.5.2; add CUDA gencode
--   MS-81 fix faiss ptx issue; change cuda gencode
+-   MS-81 Fix faiss ptx issue; change cuda gencode
 -   MS-84 cmake: add arrow, jemalloc and jsoncons third party; default build option OFF
 -   MS-85 add NetIO metric
 -   MS-96 add new query interface for specified files
@@ -698,7 +698,7 @@ Please mark all change in change log and use the issue from GitHub
 -   MS-26 CMake. Add thirdparty packages
 -   MS-31 cmake: add prometheus
 -   MS-33 cmake: add -j4 to make third party packages build faster
--   MS-27 support gpu config and disable license build config in cmake
+-   MS-27 Support gpu config and disable license build config in cmake
 -   MS-47 Add query vps metrics
 -   MS-37 Add query, cache usage, disk write speed and file data size metrics
 -   MS-30 Use faiss v1.5.2

ci/jenkins/step/singleDevNightlyTest.groovy

 dir ('milvus-helm') {
-	sh 'helm version'
-	sh 'helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
-	sh 'helm repo update'
-    checkout([$class: 'GitSCM', branches: [[name: "master"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/master:refs/remotes/origin/master"]]])
+    sh 'helm version'
+    sh 'helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
+    sh 'helm repo update'
+    checkout([$class: 'GitSCM', branches: [[name: "0.8.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.8.0:refs/remotes/origin/0.8.0"]]])
     sh "helm install --wait --timeout 600s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ."
 }
 
@@ -16,7 +16,7 @@ load "ci/jenkins/step/cleanupSingleDev.groovy"
 
 if (!fileExists('milvus-helm')) {
     dir ("milvus-helm") {
-        checkout([$class: 'GitSCM', branches: [[name: "master"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/master:refs/remotes/origin/master"]]])
+        checkout([$class: 'GitSCM', branches: [[name: "0.8.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.8.0:refs/remotes/origin/0.8.0"]]])
     }
 }
 dir ("milvus-helm") {

ci/jenkins/step/singleDevTest.groovy

 dir ('milvus-helm') {
-	sh 'helm version'
-	sh 'helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
-	sh 'helm repo update'
-    checkout([$class: 'GitSCM', branches: [[name: "master"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/master:refs/remotes/origin/master"]]])
+    sh 'helm version'
+    sh 'helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
+    sh 'helm repo update'
+    checkout([$class: 'GitSCM', branches: [[name: "0.8.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.8.0:refs/remotes/origin/0.8.0"]]])
     sh "helm install --wait --timeout 600s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ."
 }
 

core/CMakeLists.txt

@@ -90,7 +90,7 @@ if (MILVUS_VERSION_MAJOR STREQUAL ""
         OR MILVUS_VERSION_MINOR STREQUAL ""
         OR MILVUS_VERSION_PATCH STREQUAL "")
     message(WARNING "Failed to determine Milvus version from git branch name")
-    set(MILVUS_VERSION "0.7.1")
+    set(MILVUS_VERSION "0.8.0")
 endif ()
 
 message(STATUS "Build version = ${MILVUS_VERSION}")

core/conf/demo/server_config.yaml

@@ -9,7 +9,7 @@
 # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 # or implied. See the License for the specific language governing permissions and limitations under the License.
 
-version: 0.2
+version: 0.3
 
 #----------------------+------------------------------------------------------------+------------+-----------------+
 # Server Config        | Description                                                | Type       | Default         |
@@ -42,8 +42,8 @@ server_config:
 #                      | Keep 'dialect://:@:/', 'dialect' can be either 'sqlite' or |            |                 |
 #                      | 'mysql', replace other texts with real values.             |            |                 |
 #----------------------+------------------------------------------------------------+------------+-----------------+
-# preload_table        | A comma-separated list of table names that need to be pre- | StringList |                 |
-#                      | loaded when Milvus server starts up.                       |            |                 |
+# preload_collection   | A comma-separated list of collection names that need to    | StringList |                 |
+#                      | be pre-loaded when Milvus server starts up.                |            |                 |
 #                      | '*' means preload all existing tables (single-quote or     |            |                 |
 #                      | double-quote required).                                    |            |                 |
 #----------------------+------------------------------------------------------------+------------+-----------------+
@@ -53,7 +53,7 @@ server_config:
 #----------------------+------------------------------------------------------------+------------+-----------------+
 db_config:
   backend_url: sqlite://:@:/
-  preload_table:
+  preload_collection:
   auto_flush_interval: 1
 
 #----------------------+------------------------------------------------------------+------------+-----------------+

core/conf/server_cpu_config.template

@@ -9,7 +9,7 @@
 # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 # or implied. See the License for the specific language governing permissions and limitations under the License.
 
-version: 0.2
+version: 0.3
 
 #----------------------+------------------------------------------------------------+------------+-----------------+
 # Server Config        | Description                                                | Type       | Default         |
@@ -42,8 +42,8 @@ server_config:
 #                      | Keep 'dialect://:@:/', 'dialect' can be either 'sqlite' or |            |                 |
 #                      | 'mysql', replace other texts with real values.             |            |                 |
 #----------------------+------------------------------------------------------------+------------+-----------------+
-# preload_table        | A comma-separated list of table names that need to be pre- | StringList |                 |
-#                      | loaded when Milvus server starts up.                       |            |                 |
+# preload_collection   | A comma-separated list of collection names that need to    | StringList |                 |
+#                      | be pre-loaded when Milvus server starts up.                |            |                 |
 #                      | '*' means preload all existing tables (single-quote or     |            |                 |
 #                      | double-quote required).                                    |            |                 |
 #----------------------+------------------------------------------------------------+------------+-----------------+
@@ -53,7 +53,7 @@ server_config:
 #----------------------+------------------------------------------------------------+------------+-----------------+
 db_config:
   backend_url: sqlite://:@:/
-  preload_table:
+  preload_collection:
   auto_flush_interval: 1
 
 #----------------------+------------------------------------------------------------+------------+-----------------+

core/conf/server_gpu_config.template

@@ -9,7 +9,7 @@
 # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 # or implied. See the License for the specific language governing permissions and limitations under the License.
 
-version: 0.2
+version: 0.3
 
 #----------------------+------------------------------------------------------------+------------+-----------------+
 # Server Config        | Description                                                | Type       | Default         |
@@ -42,8 +42,8 @@ server_config:
 #                      | Keep 'dialect://:@:/', 'dialect' can be either 'sqlite' or |            |                 |
 #                      | 'mysql', replace other texts with real values.             |            |                 |
 #----------------------+------------------------------------------------------------+------------+-----------------+
-# preload_table        | A comma-separated list of table names that need to be pre- | StringList |                 |
-#                      | loaded when Milvus server starts up.                       |            |                 |
+# preload_collection   | A comma-separated list of collection names that need to    | StringList |                 |
+#                      | be pre-loaded when Milvus server starts up.                |            |                 |
 #                      | '*' means preload all existing tables (single-quote or     |            |                 |
 #                      | double-quote required).                                    |            |                 |
 #----------------------+------------------------------------------------------------+------------+-----------------+
@@ -53,7 +53,7 @@ server_config:
 #----------------------+------------------------------------------------------------+------------+-----------------+
 db_config:
   backend_url: sqlite://:@:/
-  preload_table:
+  preload_collection:
   auto_flush_interval: 1
 
 #----------------------+------------------------------------------------------------+------------+-----------------+

core/src/config/Config.cpp

@@ -39,9 +39,8 @@ namespace server {
 
 constexpr int64_t GB = 1UL << 30;
 
-static const std::unordered_map<std::string, std::string> milvus_config_version_map({{"0.6.0", "0.1"},
-                                                                                     {"0.7.0", "0.2"},
-                                                                                     {"0.7.1", "0.2"}});
+static const std::unordered_map<std::string, std::string> milvus_config_version_map(
+    {{"0.6.0", "0.1"}, {"0.7.0", "0.2"}, {"0.7.1", "0.2"}, {"0.8.0", "0.3"}});
 
 /////////////////////////////////////////////////////////////
 Config::Config() {
@@ -133,8 +132,8 @@ Config::ValidateConfig() {
     std::string db_backend_url;
     CONFIG_CHECK(GetDBConfigBackendUrl(db_backend_url));
 
-    std::string db_preload_table;
-    CONFIG_CHECK(GetDBConfigPreloadCollection(db_preload_table));
+    std::string db_preload_collection;
+    CONFIG_CHECK(GetDBConfigPreloadCollection(db_preload_collection));
 
     int64_t db_archive_disk_threshold;
     CONFIG_CHECK(GetDBConfigArchiveDiskThreshold(db_archive_disk_threshold));
@@ -777,7 +776,7 @@ Config::CheckDBConfigBackendUrl(const std::string& value) {
 
 Status
 Config::CheckDBConfigPreloadCollection(const std::string& value) {
-    fiu_return_on("check_config_preload_table_fail", Status(SERVER_INVALID_ARGUMENT, ""));
+    fiu_return_on("check_config_preload_collection_fail", Status(SERVER_INVALID_ARGUMENT, ""));
 
     if (value.empty() || value == "*") {
         return Status::OK();
@@ -803,7 +802,7 @@ Config::CheckDBConfigPreloadCollection(const std::string& value) {
     if (table_set.size() != tables.size()) {
         std::string msg =
             "Invalid preload tables. "
-            "Possible reason: db_config.preload_table contains duplicate collection.";
+            "Possible reason: db_config.preload_collection contains duplicate collection.";
         return Status(SERVER_INVALID_ARGUMENT, msg);
     }
 

core/src/config/Config.h

@@ -57,7 +57,7 @@ static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD = "archive_disk_threshold";
 static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT = "0";
 static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD = "archive_days_threshold";
 static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0";
-static const char* CONFIG_DB_PRELOAD_COLLECTION = "preload_table";
+static const char* CONFIG_DB_PRELOAD_COLLECTION = "preload_collection";
 static const char* CONFIG_DB_PRELOAD_COLLECTION_DEFAULT = "";
 static const char* CONFIG_DB_AUTO_FLUSH_INTERVAL = "auto_flush_interval";
 static const char* CONFIG_DB_AUTO_FLUSH_INTERVAL_DEFAULT = "1";

core/src/server/DBWrapper.cpp

@@ -205,16 +205,16 @@ DBWrapper::StartService() {
     db_->Start();
 
     // preload collection
-    std::string preload_tables;
-    s = config.GetDBConfigPreloadCollection(preload_tables);
+    std::string preload_collections;
+    s = config.GetDBConfigPreloadCollection(preload_collections);
     if (!s.ok()) {
         std::cerr << s.ToString() << std::endl;
         return s;
     }
 
-    s = PreloadCollections(preload_tables);
+    s = PreloadCollections(preload_collections);
     if (!s.ok()) {
-        std::cerr << "ERROR! Failed to preload tables: " << preload_tables << std::endl;
+        std::cerr << "ERROR! Failed to preload tables: " << preload_collections << std::endl;
         std::cerr << s.ToString() << std::endl;
         kill(0, SIGUSR1);
     }
@@ -232,10 +232,10 @@ DBWrapper::StopService() {
 }
 
 Status
-DBWrapper::PreloadCollections(const std::string& preload_tables) {
-    if (preload_tables.empty()) {
+DBWrapper::PreloadCollections(const std::string& preload_collections) {
+    if (preload_collections.empty()) {
         // do nothing
-    } else if (preload_tables == "*") {
+    } else if (preload_collections == "*") {
         // load all tables
         std::vector<engine::meta::CollectionSchema> table_schema_array;
         db_->AllCollections(table_schema_array);
@@ -248,7 +248,7 @@ DBWrapper::PreloadCollections(const std::string& preload_tables) {
         }
     } else {
         std::vector<std::string> collection_names;
-        StringHelpFunctions::SplitStringByDelimeter(preload_tables, ",", collection_names);
+        StringHelpFunctions::SplitStringByDelimeter(preload_collections, ",", collection_names);
         for (auto& name : collection_names) {
             auto status = db_->PreloadCollection(name);
             if (!status.ok()) {

core/src/server/DBWrapper.h

@@ -48,7 +48,7 @@ class DBWrapper {
 
  private:
     Status
-    PreloadCollections(const std::string& preload_tables);
+    PreloadCollections(const std::string& preload_collections);
 
  private:
     engine::DBPtr db_;

core/src/server/web_impl/README.md

@@ -1388,7 +1388,7 @@ $ curl -X GET "http://127.0.0.1:19121/system/version" -H "accept: application/js
 ##### Response
 
 ```json
-{"code":0,"message":"OK","reply": "0.7.1" }
+{"code":0,"message":"OK","reply": "0.8.0" }
 ```
 
 ### `system/{op}` (PUT)

core/unittest/server/test_config.cpp

@@ -964,10 +964,10 @@ TEST_F(ConfigTest, SERVER_CONFIG_RESET_DEFAULT_CONFIG_FAIL_TEST) {
     ASSERT_FALSE(s.ok());
     fiu_disable("check_config_backend_url_fail");
 
-    fiu_enable("check_config_preload_table_fail", 1, NULL, 0);
+    fiu_enable("check_config_preload_collection_fail", 1, NULL, 0);
     s = config.ResetDefaultConfig();
     ASSERT_FALSE(s.ok());
-    fiu_disable("check_config_preload_table_fail");
+    fiu_disable("check_config_preload_collection_fail");
 
     fiu_enable("check_config_archive_disk_threshold_fail", 1, NULL, 0);
     s = config.ResetDefaultConfig();

core/unittest/server/test_web.cpp

@@ -523,7 +523,7 @@ static const char* CONTROLLER_TEST_VALID_CONFIG_STR =
     "-+\n"
     "db_config:\n"
     "  backend_url: sqlite://:@:/\n"
-    "  preload_table:\n"
+    "  preload_collection:\n"
     "\n"
     "#----------------------+------------------------------------------------------------+------------+----------------"
     "-+\n"
@@ -1660,11 +1660,11 @@ TEST_F(WebControllerTest, CONFIG) {
     OString collection_name_s = "milvus_test_webcontroller_test_preload_collection_s";
     GenTable(client_ptr, conncetion_ptr, collection_name_s, 16, 10, "L2");
 
-    OString body_str = "{\"db_config\": {\"preload_table\": \"" + collection_name + "\"}}";
+    OString body_str = "{\"db_config\": {\"preload_collection\": \"" + collection_name + "\"}}";
     response = client_ptr->op("config", body_str, conncetion_ptr);
     ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str();
 
-    body_str = "{\"db_config\": {\"preload_table\": \"" + collection_name + "," + collection_name_s + "\"}}";
+    body_str = "{\"db_config\": {\"preload_collection\": \"" + collection_name + "," + collection_name_s + "\"}}";
     response = client_ptr->op("config", body_str, conncetion_ptr);
     ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str();
     auto set_result_json = nlohmann::json::parse(response->readBodyToString()->c_str());

core/unittest/server/utils.cpp

@@ -22,7 +22,7 @@ namespace {
 static const char* VALID_CONFIG_STR =
     "# Default values are used when you make no changes to the following parameters.\n"
     "\n"
-    "version: 0.2"
+    "version: 0.3"
     "\n"
     "server_config:\n"
     "  address: 0.0.0.0                  # milvus server ip address (IPv4)\n"
@@ -32,7 +32,7 @@ static const char* VALID_CONFIG_STR =
     "\n"
     "db_config:\n"
     "  backend_url: sqlite://:@:/        \n"
-    "  preload_table:                    \n"
+    "  preload_collection:                    \n"
     "\n"
     "storage_config:\n"
     "  primary_path: /tmp/milvus         # path used to store data and meta\n"

sdk/grpc/ClientProxy.cpp

@@ -17,7 +17,7 @@
 
 #include "grpc-gen/gen-milvus/milvus.grpc.pb.h"
 
-#define MILVUS_SDK_VERSION "0.7.1";
+#define MILVUS_SDK_VERSION "0.8.0";
 
 namespace milvus {
 

tests/milvus_benchmark/conf/server_config.yaml

@@ -9,7 +9,7 @@ db_config:
   secondary_path:                   
   backend_url: sqlite://:@:/      
   insert_buffer_size: 4             
-  preload_table:        
+  preload_collection:
 
 metric_config:
   enable_monitor: true            

tests/milvus_benchmark/conf/server_config.yaml.cpu

@@ -10,7 +10,7 @@ db_config:
   backend_url: sqlite://:@:/    
   insert_buffer_size: 4               
   build_index_gpu: 0  
-  preload_table:             
+  preload_collection:
 
 metric_config:
   enable_monitor: false             

tests/milvus_benchmark/conf/server_config.yaml.multi

@@ -10,7 +10,7 @@ db_config:
   backend_url: sqlite://:@:/    
   insert_buffer_size: 4               
   build_index_gpu: 0  
-  preload_table:             
+  preload_collection:
 
 metric_config:
   enable_monitor: false             

tests/milvus_benchmark/conf/server_config.yaml.single

@@ -10,7 +10,7 @@ db_config:
   backend_url: sqlite://:@:/    
   insert_buffer_size: 4               
   build_index_gpu: 0  
-  preload_table:       
+  preload_collection:
 
 metric_config:
   enable_monitor: false             

tests/milvus_python_test/test_config.py

@@ -1278,25 +1278,25 @@ class TestDBConfig:
         assert status.OK()
 
     @pytest.mark.level(2)
-    def test_get_preload_table_invalid_child_key(self, connect, collection):
+    def test_get_preload_collection_invalid_child_key(self, connect, collection):
         '''
         target: get invalid child key
-        method: call get_config without child_key: preload_table
+        method: call get_config without child_key: preload_collection
         expected: status not ok
         '''
-        invalid_configs = ["preloadtable", "preload_table "]
+        invalid_configs = ["preloadtable", "preload_collection "]
         for config in invalid_configs:
             status, config_value = connect.get_config("db_config", config)
             assert not status.OK()
 
     @pytest.mark.timeout(CONFIG_TIMEOUT)
-    def test_get_preload_table_valid(self, connect, collection):
+    def test_get_preload_collection_valid(self, connect, collection):
         '''
-        target: get preload_table
+        target: get preload_collection
         method: call get_config correctly
         expected: status ok
         '''
-        status, config_value = connect.get_config("db_config", "preload_table")
+        status, config_value = connect.get_config("db_config", "preload_collection")
         assert status.OK()
 
     @pytest.mark.level(2)
@@ -1350,15 +1350,15 @@ class TestDBConfig:
         assert status.OK()
         assert config_value == 'sqlite://:@:/'
 
-    def test_set_preload_table_valid(self, connect, collection):
+    def test_set_preload_collection_valid(self, connect, collection):
         '''
-        target: set preload_table
+        target: set preload_collection
         method: call set_config correctly
         expected: status ok, set successfully
         '''
-        status, reply = connect.set_config("db_config", "preload_table", "")
+        status, reply = connect.set_config("db_config", "preload_collection", "")
         assert status.OK()
-        status, config_value = connect.get_config("db_config", "preload_table")
+        status, config_value = connect.get_config("db_config", "preload_collection")
         assert status.OK()
         assert config_value == ""
 

tests/milvus_python_test/test_ping.py

 import logging
 import pytest
 
-__version__ = '0.7.1'
+__version__ = '0.8.0'
 
 
 class TestPing: