Merge branch '0.6.0' into issue-509

CHANGELOG.md

@@ -22,6 +22,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - \#458 - Index data is not compatible between 0.5 and 0.6
 - \#486 - gpu no usage during index building
 - \#509 - IVF_PQ index build trapped into dead loop caused by invalid params
+- \#513 - Unittest DELETE_BY_RANGE sometimes failed
 
 ## Feature
 - \#12 - Pure CPU version for Milvus

README.md

@@ -7,6 +7,7 @@
 [![codebeat badge](https://codebeat.co/badges/e030a4f6-b126-4475-a938-4723d54ec3a7?style=plastic)](https://codebeat.co/projects/github-com-jinhai-cn-milvus-master)
 ![Release](https://img.shields.io/badge/release-v0.5.3-yellowgreen)
 ![Release_date](https://img.shields.io/badge/release%20date-November-yellowgreen)
+[![codecov](https://codecov.io/gh/milvus-io/milvus/branch/master/graph/badge.svg)](https://codecov.io/gh/milvus-io/milvus)
 
 [中文版](README_CN.md) | [日本語版](README_JP.md)
 

README_CN.md

@@ -6,6 +6,8 @@
 [![codebeat badge](https://codebeat.co/badges/e030a4f6-b126-4475-a938-4723d54ec3a7?style=plastic)](https://codebeat.co/projects/github-com-jinhai-cn-milvus-master)
 ![Release](https://img.shields.io/badge/release-v0.5.3-yellowgreen)
 ![Release_date](https://img.shields.io/badge/release_date-October-yellowgreen)
+[![codecov](https://codecov.io/gh/milvus-io/milvus/branch/master/graph/badge.svg)](https://codecov.io/gh/milvus-io/milvus)
+
 
 # 欢迎来到 Milvus
 

core/CMakeLists.txt

@@ -31,9 +31,21 @@ GET_CURRENT_TIME(BUILD_TIME)
 string(REGEX REPLACE "\n" "" BUILD_TIME ${BUILD_TIME})
 message(STATUS "Build time = ${BUILD_TIME}")
 
+if (NOT DEFINED CMAKE_BUILD_TYPE)
+    set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build.")
+endif ()
+
+set (GIT_BRANCH_NAME_REGEX "[0-9]+\\.[0-9]+\\.[0-9]")
+
 MACRO(GET_GIT_BRANCH_NAME GIT_BRANCH_NAME)
     execute_process(COMMAND sh "-c" "git log --decorate | head -n 1 | sed 's/.*(\\(.*\\))/\\1/' | sed 's/.* \\(.*\\),.*/\\1/' | sed 's=[a-zA-Z]*\/==g'"
             OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
+    if(NOT GIT_BRANCH_NAME MATCHES "${GIT_BRANCH_NAME_REGEX}")
+        execute_process(COMMAND "git" rev-parse --abbrev-ref HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
+    endif ()
+    if(NOT GIT_BRANCH_NAME MATCHES "${GIT_BRANCH_NAME_REGEX}")
+        execute_process(COMMAND "git" symbolic-ref --short -q HEAD HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
+    endif ()
 ENDMACRO(GET_GIT_BRANCH_NAME)
 
 GET_GIT_BRANCH_NAME(GIT_BRANCH_NAME)
@@ -43,7 +55,7 @@ if (NOT GIT_BRANCH_NAME STREQUAL "")
 endif ()
 
 set(MILVUS_VERSION "${GIT_BRANCH_NAME}")
-string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]" MILVUS_VERSION "${MILVUS_VERSION}")
+string(REGEX MATCH "${GIT_BRANCH_NAME_REGEX}" MILVUS_VERSION "${MILVUS_VERSION}")
 
 if (CMAKE_BUILD_TYPE STREQUAL "Release")
     set(BUILD_TYPE "Release")

core/src/db/DBImpl.cpp

@@ -105,7 +105,8 @@ DBImpl::Stop() {
     shutting_down_.store(true, std::memory_order_release);
 
     // makesure all memory data serialized
-    MemSerialize();
+    std::set<std::string> sync_table_ids;
+    SyncMemData(sync_table_ids);
 
     // wait compaction/buildindex finish
     bg_timer_thread_.join();
@@ -329,7 +330,10 @@ DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) {
         return SHUTDOWN_ERROR;
     }
 
-    Status status;
+    // serialize memory data
+    std::set<std::string> sync_table_ids;
+    auto status = SyncMemData(sync_table_ids);
+
     {
         std::unique_lock<std::mutex> lock(build_index_mutex_);
 
@@ -588,12 +592,12 @@ DBImpl::StartMetricTask() {
 }
 
 Status
-DBImpl::MemSerialize() {
+DBImpl::SyncMemData(std::set<std::string>& sync_table_ids) {
     std::lock_guard<std::mutex> lck(mem_serialize_mutex_);
     std::set<std::string> temp_table_ids;
     mem_mgr_->Serialize(temp_table_ids);
     for (auto& id : temp_table_ids) {
-        compact_table_ids_.insert(id);
+        sync_table_ids.insert(id);
     }
 
     if (!temp_table_ids.empty()) {
@@ -612,7 +616,7 @@ DBImpl::StartCompactionTask() {
     }
 
     // serialize memory data
-    MemSerialize();
+    SyncMemData(compact_table_ids_);
 
     // compactiong has been finished?
     {

core/src/db/DBImpl.h

@@ -150,7 +150,7 @@ class DBImpl : public DB {
     BackgroundBuildIndex();
 
     Status
-    MemSerialize();
+    SyncMemData(std::set<std::string>& sync_table_ids);
 
     Status
     GetFilesToBuildIndex(const std::string& table_id, const std::vector<int>& file_types,

core/src/index/cmake/BuildUtilsCore.cmake

@@ -74,7 +74,7 @@ function(ExternalProject_Use_Cache project_name package_file install_path)
             ${CMAKE_COMMAND} -E echo
             "Extracting ${package_file} to ${install_path}"
             COMMAND
-            ${CMAKE_COMMAND} -E tar xzvf ${package_file} ${install_path}
+            ${CMAKE_COMMAND} -E tar xzf ${package_file} ${install_path}
             WORKING_DIRECTORY ${INDEX_BINARY_DIR}
             )
 

tests/milvus_python_test/test_index.py

@@ -1208,9 +1208,9 @@ class TestCreateIndexParamsInvalid(object):
         nlist = index_params["nlist"]
         logging.getLogger().info(index_params)
         status, ids = connect.add_vectors(table, vectors)
-        # if not isinstance(index_type, int) or not isinstance(nlist, int):
-        with pytest.raises(Exception) as e:
+        if (not index_type) or (not nlist) or (not isinstance(index_type, IndexType)) or (not isinstance(nlist, int)):
+            with pytest.raises(Exception) as e:
+                status = connect.create_index(table, index_params)
+        else:
             status = connect.create_index(table, index_params)
-        # else:
-        #     status = connect.create_index(table, index_params)
-        #     assert not status.OK()
+            assert not status.OK()

tests/milvus_python_test/utils.py

@@ -54,12 +54,12 @@ def gen_long_str(num):
 
 def gen_invalid_ips():
     ips = [
-            "255.0.0.0",
-            "255.255.0.0",
-            "255.255.255.0",
-            "255.255.255.255",
+            # "255.0.0.0",
+            # "255.255.0.0",
+            # "255.255.255.0",
+            # "255.255.255.255",
             "127.0.0",
-            "123.0.0.2",
+            # "123.0.0.2",
             "12-s",
             " ",
             "12 s",
@@ -114,12 +114,12 @@ def gen_invalid_uris():
 
             # invalid ip
             "tcp:// :%s" % port,
-            "tcp://123.0.0.1:%s" % port,
+            # "tcp://123.0.0.1:%s" % port,
             "tcp://127.0.0:%s" % port,
             "tcp://255.0.0.0:%s" % port,
-            "tcp://255.255.0.0:%s" % port,
-            "tcp://255.255.255.0:%s" % port,
-            "tcp://255.255.255.255:%s" % port,
+            # "tcp://255.255.0.0:%s" % port,
+            # "tcp://255.255.255.0:%s" % port,
+            # "tcp://255.255.255.255:%s" % port,
             "tcp://\n:%s" % port,
 
     ]
@@ -263,9 +263,7 @@ def gen_invalid_index_types():
 
 def gen_invalid_nlists():
     nlists = [
-            0,
             -1,
-            1000000000000001,
             # None,
             [1,2,3],
             (1,2),
@@ -549,4 +547,4 @@ if __name__ == "__main__":
         p.join()
     time.sleep(3)
     status, count = milvus.get_table_row_count(table)
-    assert count == process_num * loop_num
\ No newline at end of file
+    assert count == process_num * loop_num