Merge remote-tracking branch 'upstream/0.6.0' into 0.6.0

CHANGELOG.md

@@ -13,6 +13,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - \#226 - Experimental shards middleware for Milvus
 
 ## Improvement
+- \#275 - Rename C++ SDK IndexType
 - \#284 - Change C++ SDK to shared library
 - \#260 - C++ SDK README
 

ci/jenkins/scripts/build.sh

-#!/bin/bash
-
-SOURCE="${BASH_SOURCE[0]}"
-while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
-  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
-  SOURCE="$(readlink "$SOURCE")"
-  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
-done
-SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
-
-CMAKE_BUILD_DIR="${SCRIPTS_DIR}/../../../core/cmake_build"
-BUILD_TYPE="Debug"
-BUILD_UNITTEST="OFF"
-INSTALL_PREFIX="/opt/milvus"
-BUILD_COVERAGE="OFF"
-DB_PATH="/opt/milvus"
-PROFILING="OFF"
-USE_JFROG_CACHE="OFF"
-RUN_CPPLINT="OFF"
-CUSTOMIZATION="OFF" # default use ori faiss
-CUDA_COMPILER=/usr/local/cuda/bin/nvcc
-
-CUSTOMIZED_FAISS_URL="${FAISS_URL:-NONE}"
-wget -q --method HEAD ${CUSTOMIZED_FAISS_URL}
-if [ $? -eq 0 ]; then
-  CUSTOMIZATION="ON"
-else
-  CUSTOMIZATION="OFF"
-fi
-
-while getopts "o:d:t:ulcgjhx" arg
-do
-        case $arg in
-             o)
-                INSTALL_PREFIX=$OPTARG
-                ;;
-             d)
-                DB_PATH=$OPTARG
-                ;;
-             t)
-                BUILD_TYPE=$OPTARG # BUILD_TYPE
-                ;;
-             u)
-                echo "Build and run unittest cases" ;
-                BUILD_UNITTEST="ON";
-                ;;
-             l)
-                RUN_CPPLINT="ON"
-                ;;
-             c)
-                BUILD_COVERAGE="ON"
-                ;;
-             g)
-                PROFILING="ON"
-                ;;
-             j)
-                USE_JFROG_CACHE="ON"
-                ;;
-             x)
-                CUSTOMIZATION="OFF" # force use ori faiss
-                ;;
-             h) # help
-                echo "
-
-parameter:
--o: install prefix(default: /opt/milvus)
--d: db data path(default: /opt/milvus)
--t: build type(default: Debug)
--u: building unit test options(default: OFF)
--l: run cpplint, clang-format and clang-tidy(default: OFF)
--c: code coverage(default: OFF)
--g: profiling(default: OFF)
--j: use jfrog cache build directory(default: OFF)
--h: help
-
-usage:
-./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} [-u] [-l] [-r] [-c] [-g] [-j] [-h]
-                "
-                exit 0
-                ;;
-             ?)
-                echo "ERROR! unknown argument"
-        exit 1
-        ;;
-        esac
-done
-
-if [[ ! -d ${CMAKE_BUILD_DIR} ]]; then
-    mkdir ${CMAKE_BUILD_DIR}
-fi
-
-cd ${CMAKE_BUILD_DIR}
-
-# remove make cache since build.sh -l use default variables
-# force update the variables each time
-make rebuild_cache
-
-CMAKE_CMD="cmake \
--DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
--DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}
--DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
--DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
--DBUILD_COVERAGE=${BUILD_COVERAGE} \
--DMILVUS_DB_PATH=${DB_PATH} \
--DMILVUS_ENABLE_PROFILING=${PROFILING} \
--DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
--DCUSTOMIZATION=${CUSTOMIZATION} \
--DFAISS_URL=${CUSTOMIZED_FAISS_URL} \
-.."
-echo ${CMAKE_CMD}
-${CMAKE_CMD}
-
-if [[ ${RUN_CPPLINT} == "ON" ]]; then
-    # cpplint check
-    make lint
-    if [ $? -ne 0 ]; then
-        echo "ERROR! cpplint check failed"
-        exit 1
-    fi
-    echo "cpplint check passed!"
-
-    # clang-format check
-    make check-clang-format
-    if [ $? -ne 0 ]; then
-        echo "ERROR! clang-format check failed"
-        exit 1
-    fi
-    echo "clang-format check passed!"
-
-#    # clang-tidy check
-#    make check-clang-tidy
-#    if [ $? -ne 0 ]; then
-#        echo "ERROR! clang-tidy check failed"
-#        rm -f CMakeCache.txt
-#        exit 1
-#    fi
-#    echo "clang-tidy check passed!"
-else
-    # compile and build
-    make -j8 || exit 1
-    make install || exit 1
-fi

ci/jenkins/step/build.groovy

 timeout(time: 60, unit: 'MINUTES') {
-    dir ("ci/jenkins/scripts") {
-        sh "./build.sh -l"
+    dir ("ci/scripts") {
         withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
-            sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -d /opt/milvus -j -u -c"
+            sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -g -j -u -c"
         }
     }
 }

ci/jenkins/step/coverage.groovy

 timeout(time: 30, unit: 'MINUTES') {
-    dir ("ci/jenkins/scripts") {
+    dir ("ci/scripts") {
         sh "./coverage.sh -o /opt/milvus -u root -p 123456 -t \$POD_IP"
         // Set some env variables so codecov detection script works correctly
         withCredentials([[$class: 'StringBinding', credentialsId: "${env.PIPELINE_NAME}-codecov-token", variable: 'CODECOV_TOKEN']]) {

ci/scripts/build.sh

@@ -18,7 +18,7 @@ INSTALL_PREFIX="/opt/milvus"
 BUILD_COVERAGE="OFF"
 USE_JFROG_CACHE="OFF"
 RUN_CPPLINT="OFF"
-CPU_VERSION="ON"
+GPU_VERSION="OFF"
 WITH_MKL="OFF"
 CUDA_COMPILER=/usr/local/cuda/bin/nvcc
 
@@ -35,7 +35,7 @@ do
                 CORE_BUILD_DIR=$OPTARG # CORE_BUILD_DIR
                 ;;
              g)
-                CPU_VERSION="OFF";
+                GPU_VERSION="ON";
                 ;;
              u)
                 echo "Build and run unittest cases" ;
@@ -84,13 +84,13 @@ if [[ ! -d ${CORE_BUILD_DIR} ]]; then
     mkdir ${CORE_BUILD_DIR}
 fi
 
-pushd ${CORE_BUILD_DIR}
+cd ${CORE_BUILD_DIR}
 
 CMAKE_CMD="cmake \
 -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}
 -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
 -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
--DMILVUS_CPU_VERSION=${CPU_VERSION} \
+-DMILVUS_GPU_VERSION=${GPU_VERSION} \
 -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
 -DBUILD_COVERAGE=${BUILD_COVERAGE} \
 -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
@@ -100,6 +100,34 @@ ${MILVUS_CORE_DIR}"
 echo ${CMAKE_CMD}
 ${CMAKE_CMD}
 
+
+if [[ ${RUN_CPPLINT} == "ON" ]]; then
+    # cpplint check
+    make lint
+    if [ $? -ne 0 ]; then
+        echo "ERROR! cpplint check failed"
+        exit 1
+    fi
+    echo "cpplint check passed!"
+
+    # clang-format check
+    make check-clang-format
+    if [ $? -ne 0 ]; then
+        echo "ERROR! clang-format check failed"
+        exit 1
+    fi
+    echo "clang-format check passed!"
+
+#    # clang-tidy check
+#    make check-clang-tidy
+#    if [ $? -ne 0 ]; then
+#        echo "ERROR! clang-tidy check failed"
+#        rm -f CMakeCache.txt
+#        exit 1
+#    fi
+#    echo "clang-tidy check passed!"
+fi
+
 # compile and build
 make -j8 || exit 1
 make install || exit 1

ci/jenkins/scripts/coverage.sh → ci/scripts/coverage.sh

@@ -9,18 +9,22 @@ done
 SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
 
 INSTALL_PREFIX="/opt/milvus"
-CMAKE_BUILD_DIR="${SCRIPTS_DIR}/../../../core/cmake_build"
+MILVUS_CORE_DIR="${SCRIPTS_DIR}/../../core"
+CORE_BUILD_DIR="${MILVUS_CORE_DIR}/cmake_build"
 MYSQL_USER_NAME=root
 MYSQL_PASSWORD=123456
 MYSQL_HOST='127.0.0.1'
 MYSQL_PORT='3306'
 
-while getopts "o:u:p:t:h" arg
+while getopts "o:b:u:p:t:h" arg
 do
         case $arg in
              o)
                 INSTALL_PREFIX=$OPTARG
                 ;;
+             b)
+                CORE_BUILD_DIR=$OPTARG # CORE_BUILD_DIR
+                ;;
              u)
                 MYSQL_USER_NAME=$OPTARG
                 ;;
@@ -35,13 +39,14 @@ do
 
 parameter:
 -o: milvus install prefix(default: /opt/milvus)
+-b: core code build directory
 -u: mysql account
 -p: mysql password
 -t: mysql host
 -h: help
 
 usage:
-./coverage.sh -o \${INSTALL_PREFIX} -u \${MYSQL_USER} -p \${MYSQL_PASSWORD} -t \${MYSQL_HOST} [-h]
+./coverage.sh -o \${INSTALL_PREFIX} -b \${CORE_BUILD_DIR} -u \${MYSQL_USER} -p \${MYSQL_PASSWORD} -t \${MYSQL_HOST} [-h]
                 "
                 exit 0
                 ;;
@@ -63,12 +68,14 @@ FILE_INFO_OUTPUT="output.info"
 FILE_INFO_OUTPUT_NEW="output_new.info"
 DIR_LCOV_OUTPUT="lcov_out"
 
-DIR_GCNO="${CMAKE_BUILD_DIR}"
+DIR_GCNO="${CORE_BUILD_DIR}"
 DIR_UNITTEST="${INSTALL_PREFIX}/unittest"
 
+cd ${SCRIPTS_DIR}
+
 # delete old code coverage info files
-rm -rf lcov_out
-rm -f FILE_INFO_BASE FILE_INFO_MILVUS FILE_INFO_OUTPUT FILE_INFO_OUTPUT_NEW
+rm -rf ${DIR_LCOV_OUTPUT}
+rm -f ${FILE_INFO_BASE} ${FILE_INFO_MILVUS} ${FILE_INFO_OUTPUT} ${FILE_INFO_OUTPUT_NEW}
 
 MYSQL_DB_NAME=milvus_`date +%s%N`
 

core/CMakeLists.txt

@@ -174,7 +174,13 @@ add_custom_target(Clean-All COMMAND ${CMAKE_BUILD_TOOL} clean)
 if ("${MILVUS_DB_PATH}" STREQUAL "")
     set(MILVUS_DB_PATH "/tmp/milvus")
 endif ()
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml)
+
+if (MILVUS_GPU_VERSION)
+    configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_gpu_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml)
+else()
+    configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_cpu_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml)
+endif()
+
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.conf)
 
 install(DIRECTORY scripts/

core/conf/server_cpu_config.template

+# Default values are used when you make no changes to the following parameters.
+
+server_config:
+  address: 0.0.0.0                  # milvus server ip address (IPv4)
+  port: 19530                       # milvus server port, must in range [1025, 65534]
+  deploy_mode: single               # deployment type: single, cluster_readonly, cluster_writable
+  time_zone: UTC+8                  # time zone, must be in format: UTC+X
+
+db_config:
+  primary_path: @MILVUS_DB_PATH@    # path used to store data and meta
+  secondary_path:                   # path used to store data only, split by semicolon
+
+  backend_url: sqlite://:@:/        # URI format: dialect://username:password@host:port/database
+                                    # Keep 'dialect://:@:/', and replace other texts with real values
+                                    # Replace 'dialect' with 'mysql' or 'sqlite'
+
+  insert_buffer_size: 4             # GB, maximum insert buffer size allowed, must be a positive integer
+                                    # sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory
+
+  preload_table:                    # preload data at startup, '*' means load all tables, empty value means no preload
+                                    # you can specify preload tables like this: table1,table2,table3
+
+metric_config:
+  enable_monitor: false             # enable monitoring or not, must be a boolean
+  collector: prometheus             # prometheus
+  prometheus_config:
+    port: 8080                      # port prometheus uses to fetch metrics, must in range [1025, 65534]
+
+cache_config:
+
+  cpu_cache_capacity: 16            # GB, CPU memory used for cache, must be a positive integer
+  cpu_cache_threshold: 0.85         # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0]
+  cache_insert_data: false          # whether to load inserted data into cache, must be a boolean
+
+engine_config:
+  use_blas_threshold: 1100          # if nq <  use_blas_threshold, use SSE, faster with fluctuated response times
+                                    # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times
+  gpu_search_threshold: 1000        # threshold beyond which the search computation is executed on GPUs only
+
+resource_config:
+  search_resources:                 # define the device used for search computation
+    - cpu
+  index_build_device: cpu           # CPU used for building index

core/conf/server_config.template → core/conf/server_gpu_config.template

@@ -27,15 +27,12 @@ metric_config:
     port: 8080                      # port prometheus uses to fetch metrics, must in range [1025, 65534]
 
 cache_config:
-
   cpu_cache_capacity: 16            # GB, CPU memory used for cache, must be a positive integer
   cpu_cache_threshold: 0.85         # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0]
+  gpu_cache_capacity: 4             # GB, GPU memory used for cache, must be a positive integer
+  gpu_cache_threshold: 0.85         # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0]
   cache_insert_data: false          # whether to load inserted data into cache, must be a boolean
 
-#Uncomment the following config if you are using GPU version
-#  gpu_cache_capacity: 4             # GB, GPU memory used for cache, must be a positive integer
-#  gpu_cache_threshold: 0.85         # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0]
-
 engine_config:
   use_blas_threshold: 1100          # if nq <  use_blas_threshold, use SSE, faster with fluctuated response times
                                     # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times
@@ -44,4 +41,5 @@ engine_config:
 resource_config:
   search_resources:                 # define the devices used for search computation, must be in format: cpu or gpux
     - cpu
-  index_build_device: cpu           # CPU / GPU used for building index, must be in format: cpu / gpux
+    - gpu0
+  index_build_device: gpu0          # CPU / GPU used for building index, must be in format: cpu or gpux

core/src/sdk/examples/partition/src/ClientTest.cpp

@@ -41,7 +41,7 @@ constexpr int64_t NQ = 5;
 constexpr int64_t TOP_K = 10;
 constexpr int64_t NPROBE = 32;
 constexpr int64_t SEARCH_TARGET = 5000;  // change this value, result is different
-constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::gpu_ivfsq8;
+constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8;
 constexpr int32_t N_LIST = 15000;
 constexpr int32_t PARTITION_COUNT = 5;
 constexpr int32_t TARGET_PARTITION = 3;

core/src/sdk/examples/simple/src/ClientTest.cpp

@@ -41,7 +41,7 @@ constexpr int64_t TOP_K = 10;
 constexpr int64_t NPROBE = 32;
 constexpr int64_t SEARCH_TARGET = 5000;  // change this value, result is different
 constexpr int64_t ADD_VECTOR_LOOP = 5;
-constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::gpu_ivfsq8;
+constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8;
 constexpr int32_t N_LIST = 15000;
 
 milvus::TableSchema

core/src/sdk/examples/utils/Utils.cpp

@@ -89,14 +89,16 @@ Utils::MetricTypeName(const milvus::MetricType& metric_type) {
 std::string
 Utils::IndexTypeName(const milvus::IndexType& index_type) {
     switch (index_type) {
-        case milvus::IndexType::cpu_idmap:
-            return "cpu idmap";
-        case milvus::IndexType::gpu_ivfflat:
-            return "gpu ivflat";
-        case milvus::IndexType::gpu_ivfsq8:
-            return "gpu ivfsq8";
-        case milvus::IndexType::mix_nsg:
-            return "mix nsg";
+        case milvus::IndexType::FLAT:
+            return "FLAT";
+        case milvus::IndexType::IVFFLAT:
+            return "IVFFLAT";
+        case milvus::IndexType::IVFSQ8:
+            return "IVFSQ8";
+        case milvus::IndexType::NSG:
+            return "NSG";
+        case milvus::IndexType::IVFSQ8H:
+            return "IVFSQ8H";
         default:
             return "Unknown index type";
     }

core/src/sdk/include/MilvusApi.h

@@ -31,12 +31,12 @@ namespace milvus {
  * @brief Index Type
  */
 enum class IndexType {
-    invalid = 0,
-    cpu_idmap,
-    gpu_ivfflat,
-    gpu_ivfsq8,
-    mix_nsg,
-    ivfsq8h,
+    INVALID = 0,
+    FLAT = 1,
+    IVFFLAT = 2,
+    IVFSQ8 = 3,
+    NSG = 4,
+    IVFSQ8H = 5,
 };
 
 enum class MetricType {