Merge remote-tracking branch 'upstream/branch-0.4.0' into enable_nsg

Former-commit-id: 77d19051a4a9a945fc6068fa2641d0343b69ca27

ci/jenkinsfile/milvus_build.groovy

@@ -7,7 +7,7 @@ container('milvus-build-env') {
                     dir ("cpp") {
                         sh "git config --global user.email \"test@zilliz.com\""
                         sh "git config --global user.name \"test\""
-                        sh "./build.sh -t ${params.BUILD_TYPE} -u -c"
+                        sh "./build.sh -t ${params.BUILD_TYPE} -j -u -c"
                     }
                 } catch (exc) {
                     updateGitlabCommitStatus name: 'Build Engine', state: 'failed'

cpp/CHANGELOG.md

@@ -60,6 +60,7 @@ Please mark all change in change log and use the ticket from JIRA.
 ## Task
 
 - MS-125 - Create 0.3.1 release branch
+- MS-306 - Optimize build efficiency
 
 # Milvus 0.3.0 (2019-06-30)
 

cpp/build.sh

@@ -9,8 +9,10 @@ BUILD_COVERAGE="OFF"
 DB_PATH="/opt/milvus"
 PROFILING="OFF"
 BUILD_FAISS_WITH_MKL="OFF"
+USE_JFROG_CACHE="OFF"
+KNOWHERE_OPTS=""
 
-while getopts "p:d:t:uhlrcgm" arg
+while getopts "p:d:t:uhlrcgmj" arg
 do
         case $arg in
              t)
@@ -44,6 +46,10 @@ do
              m)
                 BUILD_FAISS_WITH_MKL="ON"
                 ;;
+             j)
+                USE_JFROG_CACHE="ON"
+                KNOWHERE_OPTS="${KNOWHERE_OPTS} -j"
+                ;;
              h) # help
                 echo "
 
@@ -57,9 +63,10 @@ parameter:
 -c: code coverage(default: OFF)
 -g: profiling(default: OFF)
 -m: build faiss with MKL(default: OFF)
+-j: use jfrog cache build directory
 
 usage:
-./build.sh -t \${BUILD_TYPE} [-u] [-h] [-g] [-r] [-c] [-m]
+./build.sh -t \${BUILD_TYPE} [-u] [-h] [-g] [-r] [-c] [-m] [-j]
                 "
                 exit 0
                 ;;
@@ -78,7 +85,7 @@ fi
 # Build Knowhere
 KNOWHERE_BUILD_DIR="`pwd`/thirdparty/knowhere_build"
 pushd `pwd`/thirdparty/knowhere
-./build.sh -t Release -p ${KNOWHERE_BUILD_DIR}
+./build.sh -t Release -p ${KNOWHERE_BUILD_DIR} ${KNOWHERE_OPTS}
 popd
 
 cd cmake_build
@@ -96,6 +103,7 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then
     -DMILVUS_ENABLE_PROFILING=${PROFILING} \
     -DBUILD_FAISS_WITH_MKL=${BUILD_FAISS_WITH_MKL} \
     -DKNOWHERE_BUILD_DIR=${KNOWHERE_BUILD_DIR} \
+    -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
     $@ ../"
     echo ${CMAKE_CMD}
 

cpp/cmake/BuildUtils.cmake

+# Define a function that extracts a cached package
+function(ExternalProject_Use_Cache project_name package_file install_path)
+    message(STATUS "Will use cached package file: ${package_file}")
+
+    ExternalProject_Add(${project_name}
+        DOWNLOAD_COMMAND ${CMAKE_COMMAND} -E echo
+            "No download step needed (using cached package)"
+        CONFIGURE_COMMAND ${CMAKE_COMMAND} -E echo
+            "No configure step needed (using cached package)"
+        BUILD_COMMAND ${CMAKE_COMMAND} -E echo
+            "No build step needed (using cached package)"
+        INSTALL_COMMAND ${CMAKE_COMMAND} -E echo
+            "No install step needed (using cached package)"
+    )
+
+    # We want our tar files to contain the Install/<package> prefix (not for any
+    # very special reason, only for consistency and so that we can identify them
+    # in the extraction logs) which means that we must extract them in the
+    # binary (top-level build) directory to have them installed in the right
+    # place for subsequent ExternalProjects to pick them up. It seems that the
+    # only way to control the working directory is with Add_Step!
+    ExternalProject_Add_Step(${project_name} extract
+        ALWAYS 1
+        COMMAND
+            ${CMAKE_COMMAND} -E echo
+            "Extracting ${package_file} to ${install_path}"
+        COMMAND
+            ${CMAKE_COMMAND} -E tar xzvf ${package_file} ${install_path}
+            WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    )
+
+    ExternalProject_Add_StepTargets(${project_name} extract)
+endfunction()
+
+# Define a function that to create a new cached package
+function(ExternalProject_Create_Cache project_name package_file install_path cache_username cache_password cache_path)
+    if(EXISTS ${package_file})
+        message(STATUS "Removing existing package file: ${package_file}")
+        file(REMOVE ${package_file})
+    endif()
+
+    message(STATUS "Will create cached package file: ${package_file}")
+
+    ExternalProject_Add_Step(${project_name} package
+        DEPENDEES install
+        BYPRODUCTS ${package_file}
+        COMMAND ${CMAKE_COMMAND} -E echo "Updating cached package file: ${package_file}"
+        COMMAND ${CMAKE_COMMAND} -E tar czvf ${package_file} ${install_path}
+        COMMAND ${CMAKE_COMMAND} -E echo "Uploading package file ${package_file} to ${cache_path}"
+        COMMAND curl -u${cache_username}:${cache_password} -T ${package_file} ${cache_path}
+    )
+
+    ExternalProject_Add_StepTargets(${project_name} package)
+endfunction()
+
 function(ADD_THIRDPARTY_LIB LIB_NAME)
     set(options)
     set(one_value_args SHARED_LIB STATIC_LIB)
@@ -92,4 +147,4 @@ function(ADD_THIRDPARTY_LIB LIB_NAME)
     else()
         message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}")
     endif()
-endfunction()
\ No newline at end of file
+endfunction()

cpp/src/db/DBImpl.cpp

@@ -369,6 +369,7 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date,
     meta::TableFileSchema table_file;
     table_file.table_id_ = table_id;
     table_file.date_ = date;
+    table_file.file_type_ = meta::TableFileSchema::NEW_MERGE;
     Status status = meta_ptr_->CreateTableFile(table_file);
 
     if (!status.ok()) {
@@ -529,7 +530,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
         meta::TableFileSchema table_file;
         table_file.table_id_ = file.table_id_;
         table_file.date_ = file.date_;
-        table_file.file_type_ = meta::TableFileSchema::INDEX; //for multi-db-path, distribute index file averagely to each path
+        table_file.file_type_ = meta::TableFileSchema::NEW_INDEX; //for multi-db-path, distribute index file averagely to each path
         Status status = meta_ptr_->CreateTableFile(table_file);
         if (!status.ok()) {
             ENGINE_LOG_ERROR << "Failed to create table: " << status.ToString();

cpp/src/db/DBMetaImpl.cpp

@@ -302,19 +302,49 @@ Status DBMetaImpl::DescribeTable(TableSchema &table_schema) {
 Status DBMetaImpl::HasNonIndexFiles(const std::string& table_id, bool& has) {
     has = false;
     try {
-        auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_),
+        auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_,
+                                                     &TableFileSchema::file_type_),
                                              where((c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW
                                                     or
                                                     c(&TableFileSchema::file_type_) == (int) TableFileSchema::NEW
                                                     or
+                                                    c(&TableFileSchema::file_type_) == (int) TableFileSchema::NEW_MERGE
+                                                    or
+                                                    c(&TableFileSchema::file_type_) == (int) TableFileSchema::NEW_INDEX
+                                                    or
                                                     c(&TableFileSchema::file_type_) == (int) TableFileSchema::TO_INDEX)
                                                    and c(&TableFileSchema::table_id_) == table_id
                                              ));
 
         if (selected.size() >= 1) {
             has = true;
-        } else {
-            has = false;
+
+            int raw_count = 0, new_count = 0, new_merge_count = 0, new_index_count = 0, to_index_count = 0;
+            for (auto &file : selected) {
+                switch (std::get<1>(file)) {
+                    case (int) TableFileSchema::RAW:
+                        raw_count++;
+                        break;
+                    case (int) TableFileSchema::NEW:
+                        new_count++;
+                        break;
+                    case (int) TableFileSchema::NEW_MERGE:
+                        new_merge_count++;
+                        break;
+                    case (int) TableFileSchema::NEW_INDEX:
+                        new_index_count++;
+                        break;
+                    case (int) TableFileSchema::TO_INDEX:
+                        to_index_count++;
+                        break;
+                    default:
+                        break;
+                }
+            }
+
+            ENGINE_LOG_DEBUG << "Table " << table_id << " currently has raw files:" << raw_count
+                << " new files:" << new_count << " new_merge files:" << new_merge_count
+                << " new_index files:" << new_index_count << " to_index files:" << to_index_count;
         }
 
     } catch (std::exception &e) {
@@ -389,7 +419,6 @@ Status DBMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
         MetricCollector metric;
 
         NextFileId(file_schema.file_id_);
-        file_schema.file_type_ = TableFileSchema::NEW;
         file_schema.dimension_ = table_schema.dimension_;
         file_schema.size_ = 0;
         file_schema.created_on_ = utils::GetMicroSecTimeStamp();
@@ -1031,7 +1060,11 @@ Status DBMetaImpl::CleanUp() {
         std::lock_guard<std::mutex> meta_lock(meta_mutex_);
 
         auto files = ConnectorPtr->select(columns(&TableFileSchema::id_),
-                                             where(c(&TableFileSchema::file_type_) == (int) TableFileSchema::NEW));
+                                             where(c(&TableFileSchema::file_type_) == (int) TableFileSchema::NEW
+                                                   or
+                                                   c(&TableFileSchema::file_type_) == (int) TableFileSchema::NEW_INDEX
+                                                   or
+                                                   c(&TableFileSchema::file_type_) == (int) TableFileSchema::NEW_MERGE));
 
         auto commited = ConnectorPtr->transaction([&]() mutable {
             for (auto &file : files) {

cpp/src/db/MetaTypes.h

@@ -43,6 +43,8 @@ struct TableFileSchema {
         TO_INDEX,
         INDEX,
         TO_DELETE,
+        NEW_MERGE,
+        NEW_INDEX,
     } FILE_TYPE;
 
     size_t id_ = 0;

cpp/src/db/MySQLMetaImpl.cpp

@@ -404,6 +404,8 @@ Status MySQLMetaImpl::HasNonIndexFiles(const std::string &table_id, bool &has) {
                                   "WHERE table_id = " << quote << table_id << " AND " <<
                                   "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " <<
                                   "file_type = " << std::to_string(TableFileSchema::NEW) << " OR " <<
+                                  "file_type = " << std::to_string(TableFileSchema::NEW_MERGE) << " OR " <<
+                                  "file_type = " << std::to_string(TableFileSchema::NEW_INDEX) << " OR " <<
                                   "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ")) " <<
                                   "AS " << quote << "check" << ";";
 
@@ -706,7 +708,6 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
         MetricCollector metric;
 
         NextFileId(file_schema.file_id_);
-        file_schema.file_type_ = TableFileSchema::NEW;
         file_schema.dimension_ = table_schema.dimension_;
         file_schema.size_ = 0;
         file_schema.created_on_ = utils::GetMicroSecTimeStamp();
@@ -1812,7 +1813,10 @@ Status MySQLMetaImpl::CleanUp() {
 
         if (!res.empty()) {
             ENGINE_LOG_DEBUG << "Remove table file type as NEW";
-            cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";";
+            cleanUpQuery << "DELETE FROM TableFiles WHERE file_type IN ("
+                    << std::to_string(TableFileSchema::NEW) << ","
+                    << std::to_string(TableFileSchema::NEW_MERGE) << ","
+                    << std::to_string(TableFileSchema::NEW_INDEX) << ");";
 
             ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str();
 

cpp/src/db/Utils.cpp

@@ -34,7 +34,7 @@ std::string GetTableFileParentFolder(const DBMetaOptions& options, const meta::T
     std::string target_path = options.path;
     uint64_t index = 0;
 
-    if(meta::TableFileSchema::INDEX == table_file.file_type_) {
+    if(meta::TableFileSchema::NEW_INDEX == table_file.file_type_) {
         // index file is large file and to be persisted permanently
         // we need to distribute index files to each db_path averagely
         // round robin according to a file counter

knowhere @ 5032e91e

-Subproject commit 02550a43b5146bd7976b8b2b3fc37ca885d1e880
+Subproject commit 5032e91e3d8bee1760c3f86f7eb4c857e59546ad