Merge branch 'branch-0.5.0' into '0.5.0'

#48 Config unittest failed

See merge request megasearch/milvus!770

Former-commit-id: 9f74cc2b01e09deee21a0d2fac4d7f43ffdb83d6

CHANGELOG.md

@@ -26,13 +26,14 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-653 - When config check fail, Milvus close without message
 - MS-654 - Describe index timeout when building index
 - MS-658 - Fix SQ8 Hybrid can't search
-- \#9 Change default gpu_cache_capacity to 4
 - MS-665 - IVF_SQ8H search crash when no GPU resource in search_resources
+- \#9 - Change default gpu_cache_capacity to 4
 - \#20 - C++ sdk example get grpc error 
 - \#23 - Add unittest to improve code coverage
 - \#31 - make clang-format failed after run build.sh -l
 - \#39 - Create SQ8H index hang if using github server version
 - \#30 - Some troubleshoot messages in Milvus do not provide enough information
+- \#48 - Config unittest failed
 
 ## Improvement
 - MS-552 - Add and change the easylogging library

core/unittest/server/CMakeLists.txt

@@ -67,11 +67,3 @@ target_link_libraries(test_server
         )
 
 install(TARGETS test_server DESTINATION unittest)
-
-configure_file(appendix/server_config.yaml
-        "${CMAKE_CURRENT_BINARY_DIR}/milvus/conf/server_config.yaml"
-        COPYONLY)
-
-configure_file(appendix/log_config.conf
-        "${CMAKE_CURRENT_BINARY_DIR}/milvus/conf/log_config.conf"
-        COPYONLY)

core/unittest/server/appendix/log_config.conf

-* GLOBAL:
-    FORMAT                  =   "%datetime | %level | %logger | %msg"
-    FILENAME                =   "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-global.log"
-    ENABLED                 =   true
-    TO_FILE                 =   true
-    TO_STANDARD_OUTPUT      =   false
-    SUBSECOND_PRECISION     =   3
-    PERFORMANCE_TRACKING    =   false
-    MAX_LOG_FILE_SIZE       =   209715200 ## Throw log files away after 200MB
-* DEBUG:
-    FILENAME                =   "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-debug.log"
-    ENABLED                 =   true
-* WARNING:
-    FILENAME                =   "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-warning.log"
-* TRACE:
-    FILENAME                =   "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-trace.log"
-* VERBOSE:
-    FORMAT                  =   "%datetime{%d/%M/%y} | %level-%vlevel | %msg"
-    TO_FILE                 =   false
-    TO_STANDARD_OUTPUT      =   false
-## Error logs
-* ERROR:
-    ENABLED                 =   true
-    FILENAME                =   "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-error.log"
-* FATAL:
-    ENABLED                 =   true
-    FILENAME                =   "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-fatal.log"

core/unittest/server/appendix/server_config.yaml

-# Default values are used when you make no changes to the following parameters.
-
-server_config:
-  address: 0.0.0.0                  # milvus server ip address (IPv4)
-  port: 19530                       # port range: 1025 ~ 65534
-  deploy_mode: single               # deployment type: single, cluster_readonly, cluster_writable
-  time_zone: UTC+8
-
-db_config:
-  primary_path: /tmp/milvus    # path used to store data and meta
-  secondary_path:                   # path used to store data only, split by semicolon
-
-  backend_url: sqlite://:@:/        # URI format: dialect://username:password@host:port/database
-                                    # Keep 'dialect://:@:/', and replace other texts with real values
-                                    # Replace 'dialect' with 'mysql' or 'sqlite'
-
-  insert_buffer_size: 4             # GB, maximum insert buffer size allowed
-                                    # sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory
-
-  preload_table:                    # preload data at startup, '*' means load all tables, empty value means no preload
-                                    # you can specify preload tables like this: table1,table2,table3
-
-metric_config:
-  enable_monitor: false             # enable monitoring or not
-  collector: prometheus             # prometheus
-  prometheus_config:
-    port: 8080                      # port prometheus uses to fetch metrics
-
-cache_config:
-  cpu_cache_capacity: 16            # GB, CPU memory used for cache
-  cpu_cache_threshold: 0.85         # percentage of data that will be kept when cache cleanup is triggered
-  gpu_cache_capacity: 4             # GB, GPU memory used for cache
-  gpu_cache_threshold: 0.85         # percentage of data that will be kept when cache cleanup is triggered
-  cache_insert_data: false          # whether to load inserted data into cache
-
-engine_config:
-  use_blas_threshold: 20            # if nq <  use_blas_threshold, use SSE, faster with fluctuated response times
-                                    # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times
-
-resource_config:
-  search_resources:                 # define the GPUs used for search computation, valid value: gpux
-    - gpu0
-  index_build_device: gpu0          # GPU used for building index

core/unittest/server/test_config.cpp

@@ -22,28 +22,26 @@
 #include "utils/CommonUtil.h"
 #include "utils/ValidationUtil.h"
 #include "server/Config.h"
+#include "server/utils.h"
 
 namespace {
 
-static const char *CONFIG_FILE_PATH = "./milvus/conf/server_config.yaml";
-static const char *LOG_FILE_PATH = "./milvus/conf/log_config.conf";
-
 static constexpr uint64_t KB = 1024;
 static constexpr uint64_t MB = KB * 1024;
 static constexpr uint64_t GB = MB * 1024;
 
 } // namespace
 
-TEST(ConfigTest, CONFIG_TEST) {
+TEST_F(ConfigTest, CONFIG_TEST) {
     milvus::server::ConfigMgr *config_mgr = milvus::server::YamlConfigMgr::GetInstance();
 
     milvus::Status s = config_mgr->LoadConfigFile("");
     ASSERT_FALSE(s.ok());
 
-    s = config_mgr->LoadConfigFile(LOG_FILE_PATH);
+    s = config_mgr->LoadConfigFile(INVALID_CONFIG_PATH);
     ASSERT_FALSE(s.ok());
 
-    s = config_mgr->LoadConfigFile(CONFIG_FILE_PATH);
+    s = config_mgr->LoadConfigFile(VALID_CONFIG_PATH);
     ASSERT_TRUE(s.ok());
 
     config_mgr->Print();
@@ -99,9 +97,9 @@ TEST(ConfigTest, CONFIG_TEST) {
     ASSERT_TRUE(seqs.empty());
 }
 
-TEST(ConfigTest, SERVER_CONFIG_TEST) {
+TEST_F(ConfigTest, SERVER_CONFIG_TEST) {
     milvus::server::Config &config = milvus::server::Config::GetInstance();
-    milvus::Status s = config.LoadConfigFile(CONFIG_FILE_PATH);
+    milvus::Status s = config.LoadConfigFile(VALID_CONFIG_PATH);
     ASSERT_TRUE(s.ok());
 
     s = config.ValidateConfig();

core/unittest/server/util_test.cpp → core/unittest/server/test_util.cpp

@@ -275,6 +275,11 @@ TEST(ValidationUtilTest, VALIDATE_INDEX_TEST) {
     ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexType((int)milvus::engine::EngineType::INVALID).code(),
               milvus::SERVER_INVALID_INDEX_TYPE);
     for (int i = 1; i <= (int)milvus::engine::EngineType::MAX_VALUE; i++) {
+#ifndef CUSTOMIZATION
+        if (i == (int)milvus::engine::EngineType::FAISS_IVFSQ8H) {
+            continue;
+        }
+#endif
         ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexType(i).code(), milvus::SERVER_SUCCESS);
     }
     ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexType(

core/unittest/server/utils.cpp

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "server/utils.h"
+
+#include <iostream>
+#include <thread>
+#include <string>
+#include <boost/filesystem.hpp>
+
+namespace {
+
+static const char
+    * VALID_CONFIG_STR = "# Default values are used when you make no changes to the following parameters.\n"
+                         "\n"
+                         "server_config:\n"
+                         "  address: 0.0.0.0                  # milvus server ip address (IPv4)\n"
+                         "  port: 19530                       # port range: 1025 ~ 65534\n"
+                         "  deploy_mode: single               \n"
+                         "  time_zone: UTC+8\n"
+                         "\n"
+                         "db_config:\n"
+                         "  primary_path: /tmp/milvus    # path used to store data and meta\n"
+                         "  secondary_path:                   # path used to store data only, split by semicolon\n"
+                         "\n"
+                         "  backend_url: sqlite://:@:/        \n"
+                         "\n"
+                         "  insert_buffer_size: 4             # GB, maximum insert buffer size allowed\n"
+                         "  preload_table:                    \n"
+                         "\n"
+                         "metric_config:\n"
+                         "  enable_monitor: false             # enable monitoring or not\n"
+                         "  collector: prometheus             # prometheus\n"
+                         "  prometheus_config:\n"
+                         "    port: 8080                      # port prometheus uses to fetch metrics\n"
+                         "\n"
+                         "cache_config:\n"
+                         "  cpu_cache_capacity: 16            # GB, CPU memory used for cache\n"
+                         "  cpu_cache_threshold: 0.85         \n"
+                         "  gpu_cache_capacity: 4             # GB, GPU memory used for cache\n"
+                         "  gpu_cache_threshold: 0.85         \n"
+                         "  cache_insert_data: false          # whether to load inserted data into cache\n"
+                         "\n"
+                         "engine_config:\n"
+                         "  use_blas_threshold: 20            \n"
+                         "\n"
+                         "resource_config:\n"
+                         "  search_resources:                 \n"
+                         "    - gpu0\n"
+                         "  index_build_device: gpu0          # GPU used for building index";
+
+static const char* INVALID_CONFIG_STR = "*INVALID*";
+
+void
+WriteToFile(const char* file_path, const char* content) {
+    boost::filesystem::path fpath(file_path);
+    boost::filesystem::fstream fstream(fpath, std::ios_base::out);
+
+    //write data to file
+    fstream << content;
+    fstream.close();
+}
+
+} // namespace
+
+
+void
+ConfigTest::SetUp() {
+    WriteToFile(VALID_CONFIG_PATH, VALID_CONFIG_STR);
+    WriteToFile(INVALID_CONFIG_PATH, INVALID_CONFIG_STR);
+}
+
+void
+ConfigTest::TearDown() {
+    boost::filesystem::remove(VALID_CONFIG_PATH);
+    boost::filesystem::remove(INVALID_CONFIG_PATH);
+}

core/unittest/server/utils.h

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#pragma once
+
+#include <gtest/gtest.h>
+#include <chrono>
+
+static const char *VALID_CONFIG_PATH = "/tmp/milvus_test/valid_config.yaml";
+static const char *INVALID_CONFIG_PATH = "/tmp/milvus_test/invalid_config.conf";
+
+class ConfigTest : public ::testing::Test {
+ protected:
+    void SetUp() override;
+    void TearDown() override;
+};