#127 Support new Index type IVFPQ

.gitignore

@@ -26,3 +26,6 @@ cmake_build
 *.lo
 *.tar.gz
 *.log
+.coverage
+*.pyc
+cov_html/

.travis.yml

+language: cpp
+sudo: required
+dist: bionic
+
+cache:
+  directories:
+    - $HOME/.ccache
+
+addons:
+  apt:
+    update: true
+
+before_install:
+  - source ci/travis/before-install.sh
+
+install:
+  - source $TRAVIS_BUILD_DIR/ci/travis/install_dependency.sh
+
+script:
+  - $TRAVIS_BUILD_DIR/ci/travis/travis_build.sh

CHANGELOG.md

@@ -8,7 +8,6 @@ Please mark all change in change log and use the ticket from JIRA.
 
 ## Feature
 - \#12 - Pure CPU version for Milvus
-- #127 - Support new Index type IVFPQ
 
 ## Improvement
 

ci/jenkins/step/build.groovy

 timeout(time: 60, unit: 'MINUTES') {
-    dir ("ci/jenkins/scripts") {
-        sh "./build.sh -l"
+    dir ("ci/scripts") {
         withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
-            sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -d /opt/milvus -j -u -c"
+            sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -g -j -u -c"
         }
     }
 }

ci/jenkins/step/coverage.groovy

 timeout(time: 30, unit: 'MINUTES') {
-    dir ("ci/jenkins/scripts") {
+    dir ("ci/scripts") {
         sh "./coverage.sh -o /opt/milvus -u root -p 123456 -t \$POD_IP"
         // Set some env variables so codecov detection script works correctly
         withCredentials([[$class: 'StringBinding', credentialsId: "${env.PIPELINE_NAME}-codecov-token", variable: 'CODECOV_TOKEN']]) {

ci/jenkins/scripts/build.sh → ci/scripts/build.sh

 #!/bin/bash
 
+set -e
+
 SOURCE="${BASH_SOURCE[0]}"
 while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
   DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
@@ -8,38 +10,37 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
 done
 SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
 
-CMAKE_BUILD_DIR="${SCRIPTS_DIR}/../../../core/cmake_build"
+MILVUS_CORE_DIR="${SCRIPTS_DIR}/../../core"
+CORE_BUILD_DIR="${MILVUS_CORE_DIR}/cmake_build"
 BUILD_TYPE="Debug"
 BUILD_UNITTEST="OFF"
 INSTALL_PREFIX="/opt/milvus"
+FAISS_ROOT=""
 BUILD_COVERAGE="OFF"
-DB_PATH="/opt/milvus"
-PROFILING="OFF"
 USE_JFROG_CACHE="OFF"
 RUN_CPPLINT="OFF"
-CUSTOMIZATION="OFF" # default use ori faiss
+GPU_VERSION="OFF"
+WITH_MKL="OFF"
 CUDA_COMPILER=/usr/local/cuda/bin/nvcc
 
-CUSTOMIZED_FAISS_URL="${FAISS_URL:-NONE}"
-wget -q --method HEAD ${CUSTOMIZED_FAISS_URL}
-if [ $? -eq 0 ]; then
-  CUSTOMIZATION="ON"
-else
-  CUSTOMIZATION="OFF"
-fi
-
-while getopts "o:d:t:ulcgjhx" arg
+while getopts "o:t:b:f:gulcjmh" arg
 do
         case $arg in
              o)
                 INSTALL_PREFIX=$OPTARG
                 ;;
-             d)
-                DB_PATH=$OPTARG
-                ;;
              t)
                 BUILD_TYPE=$OPTARG # BUILD_TYPE
                 ;;
+             b)
+                CORE_BUILD_DIR=$OPTARG # CORE_BUILD_DIR
+                ;;
+             f)
+                FAISS_ROOT=$OPTARG # FAISS ROOT PATH
+                ;;
+             g)
+                GPU_VERSION="ON";
+                ;;
              u)
                 echo "Build and run unittest cases" ;
                 BUILD_UNITTEST="ON";
@@ -50,31 +51,30 @@ do
              c)
                 BUILD_COVERAGE="ON"
                 ;;
-             g)
-                PROFILING="ON"
-                ;;
              j)
                 USE_JFROG_CACHE="ON"
                 ;;
-             x)
-                CUSTOMIZATION="OFF" # force use ori faiss
+             m)
+                WITH_MKL="ON"
                 ;;
              h) # help
                 echo "
 
 parameter:
 -o: install prefix(default: /opt/milvus)
--d: db data path(default: /opt/milvus)
 -t: build type(default: Debug)
+-b: core code build directory
+-f: faiss root path
+-g: gpu version
 -u: building unit test options(default: OFF)
 -l: run cpplint, clang-format and clang-tidy(default: OFF)
 -c: code coverage(default: OFF)
--g: profiling(default: OFF)
 -j: use jfrog cache build directory(default: OFF)
+-m: build with MKL(default: OFF)
 -h: help
 
 usage:
-./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} [-u] [-l] [-r] [-c] [-g] [-j] [-h]
+./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-u] [-l] [-c] [-j] [-m] [-h]
                 "
                 exit 0
                 ;;
@@ -85,31 +85,29 @@ usage:
         esac
 done
 
-if [[ ! -d ${CMAKE_BUILD_DIR} ]]; then
-    mkdir ${CMAKE_BUILD_DIR}
+if [[ ! -d ${CORE_BUILD_DIR} ]]; then
+    mkdir ${CORE_BUILD_DIR}
 fi
 
-cd ${CMAKE_BUILD_DIR}
-
-# remove make cache since build.sh -l use default variables
-# force update the variables each time
-make rebuild_cache
+cd ${CORE_BUILD_DIR}
 
 CMAKE_CMD="cmake \
--DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
 -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}
 -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
 -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
+-DMILVUS_GPU_VERSION=${GPU_VERSION} \
+-DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
 -DBUILD_COVERAGE=${BUILD_COVERAGE} \
--DMILVUS_DB_PATH=${DB_PATH} \
--DMILVUS_ENABLE_PROFILING=${PROFILING} \
 -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
--DCUSTOMIZATION=${CUSTOMIZATION} \
--DFAISS_URL=${CUSTOMIZED_FAISS_URL} \
-.."
+-DFAISS_ROOT=${FAISS_ROOT} \
+-DFAISS_WITH_MKL=${WITH_MKL} \
+-DArrow_SOURCE=AUTO \
+-DFAISS_SOURCE=AUTO \
+${MILVUS_CORE_DIR}"
 echo ${CMAKE_CMD}
 ${CMAKE_CMD}
 
+
 if [[ ${RUN_CPPLINT} == "ON" ]]; then
     # cpplint check
     make lint
@@ -135,8 +133,8 @@ if [[ ${RUN_CPPLINT} == "ON" ]]; then
 #        exit 1
 #    fi
 #    echo "clang-tidy check passed!"
-else
-    # compile and build
-    make -j8 || exit 1
-    make install || exit 1
 fi
+
+# compile and build
+make -j8 || exit 1
+make install || exit 1

ci/jenkins/scripts/coverage.sh → ci/scripts/coverage.sh

@@ -9,18 +9,22 @@ done
 SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
 
 INSTALL_PREFIX="/opt/milvus"
-CMAKE_BUILD_DIR="${SCRIPTS_DIR}/../../../core/cmake_build"
+MILVUS_CORE_DIR="${SCRIPTS_DIR}/../../core"
+CORE_BUILD_DIR="${MILVUS_CORE_DIR}/cmake_build"
 MYSQL_USER_NAME=root
 MYSQL_PASSWORD=123456
 MYSQL_HOST='127.0.0.1'
 MYSQL_PORT='3306'
 
-while getopts "o:u:p:t:h" arg
+while getopts "o:b:u:p:t:h" arg
 do
         case $arg in
              o)
                 INSTALL_PREFIX=$OPTARG
                 ;;
+             b)
+                CORE_BUILD_DIR=$OPTARG # CORE_BUILD_DIR
+                ;;
              u)
                 MYSQL_USER_NAME=$OPTARG
                 ;;
@@ -35,13 +39,14 @@ do
 
 parameter:
 -o: milvus install prefix(default: /opt/milvus)
+-b: core code build directory
 -u: mysql account
 -p: mysql password
 -t: mysql host
 -h: help
 
 usage:
-./coverage.sh -o \${INSTALL_PREFIX} -u \${MYSQL_USER} -p \${MYSQL_PASSWORD} -t \${MYSQL_HOST} [-h]
+./coverage.sh -o \${INSTALL_PREFIX} -b \${CORE_BUILD_DIR} -u \${MYSQL_USER} -p \${MYSQL_PASSWORD} -t \${MYSQL_HOST} [-h]
                 "
                 exit 0
                 ;;
@@ -63,12 +68,14 @@ FILE_INFO_OUTPUT="output.info"
 FILE_INFO_OUTPUT_NEW="output_new.info"
 DIR_LCOV_OUTPUT="lcov_out"
 
-DIR_GCNO="${CMAKE_BUILD_DIR}"
+DIR_GCNO="${CORE_BUILD_DIR}"
 DIR_UNITTEST="${INSTALL_PREFIX}/unittest"
 
+cd ${SCRIPTS_DIR}
+
 # delete old code coverage info files
-rm -rf lcov_out
-rm -f FILE_INFO_BASE FILE_INFO_MILVUS FILE_INFO_OUTPUT FILE_INFO_OUTPUT_NEW
+rm -rf ${DIR_LCOV_OUTPUT}
+rm -f ${FILE_INFO_BASE} ${FILE_INFO_MILVUS} ${FILE_INFO_OUTPUT} ${FILE_INFO_OUTPUT_NEW}
 
 MYSQL_DB_NAME=milvus_`date +%s%N`
 
@@ -132,8 +139,7 @@ ${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \
     "*/src/server/Server.cpp" \
     "*/src/server/DBWrapper.cpp" \
     "*/src/server/grpc_impl/GrpcServer.cpp" \
-    "*/src/external/easyloggingpp/easylogging++.h" \
-    "*/src/external/easyloggingpp/easylogging++.cc"
+    "*/thirdparty/*"
 
 if [ $? -ne 0 ]; then
     echo "gen ${FILE_INFO_OUTPUT_NEW} failed"

ci/travis/before-install.sh

+#!/bin/bash
+
+set -ex
+
+if [[ "${TRAVIS_OS_NAME}" == "linux" ]]; then
+  export CCACHE_COMPRESS=1
+  export CCACHE_COMPRESSLEVEL=5
+  export CCACHE_COMPILERCHECK=content
+  export PATH=/usr/lib/ccache/:$PATH
+  ccache --show-stats
+fi
+
+set +ex

ci/travis/install_dependency.sh

+#!/usr/bin/env bash
+
+set -e
+
+wget -P /tmp https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
+
+sudo apt-key add /tmp/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
+
+echo "deb https://apt.repos.intel.com/mkl all main" | \
+  sudo tee /etc/apt/sources.list.d/intel-mkl.list
+
+sudo wget -O /usr/share/keyrings/apache-arrow-keyring.gpg https://dl.bintray.com/apache/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-keyring.gpg
+
+sudo tee /etc/apt/sources.list.d/apache-arrow.list <<APT_LINE
+deb [arch=amd64 signed-by=/usr/share/keyrings/apache-arrow-keyring.gpg] https://dl.bintray.com/apache/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/ $(lsb_release --codename --short) main
+deb-src [signed-by=/usr/share/keyrings/apache-arrow-keyring.gpg] https://dl.bintray.com/apache/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/ $(lsb_release --codename --short) main
+APT_LINE
+
+sudo apt-get update -qq
+
+sudo apt-get install -y -q --no-install-recommends \
+  gfortran \
+  lsb-core \
+  libtool \
+  automake \
+  ccache \
+  pkg-config \
+  libarrow-dev \
+  libjemalloc-dev \
+  libboost-serialization-dev \
+  libboost-filesystem-dev \
+  libboost-system-dev \
+  libboost-regex-dev \
+  intel-mkl-gnu-2019.5-281 \
+  intel-mkl-core-2019.5-281 \
+  libmysqlclient-dev \
+  clang-format-6.0 \
+  clang-tidy-6.0 \
+  lcov
+
+sudo ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so \
+  /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so
+
+export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/intel/compilers_and_libraries_2019.5.281/linux/mkl/lib/intel64

ci/travis/travis_build.sh

+#!/usr/bin/env bash
+
+set -ex
+
+source $TRAVIS_BUILD_DIR/ci/travis/travis_env_common.sh
+
+only_library_mode=no
+
+while true; do
+    case "$1" in
+	--only-library)
+	    only_library_mode=yes
+	    shift ;;
+	*) break ;;
+    esac
+done
+
+BUILD_COMMON_FLAGS="-t ${MILVUS_BUILD_TYPE} -o ${MILVUS_INSTALL_PREFIX} -b ${MILVUS_BUILD_DIR}"
+
+if [ $only_library_mode == "yes" ]; then
+  ${TRAVIS_BUILD_DIR}/ci/scripts/build.sh ${BUILD_COMMON_FLAGS} -m
+else
+  ${TRAVIS_BUILD_DIR}/ci/scripts/build.sh ${BUILD_COMMON_FLAGS} -m -u -c
+fi
\ No newline at end of file

ci/travis/travis_env_common.sh

+export MILVUS_CORE_DIR=${TRAVIS_BUILD_DIR}/core
+export MILVUS_BUILD_DIR=${TRAVIS_BUILD_DIR}/core/cmake_build
+export MILVUS_INSTALL_PREFIX=/opt/milvus
+export MILVUS_TRAVIS_COVERAGE=${MILVUS_TRAVIS_COVERAGE:=0}
+
+if [ "${MILVUS_TRAVIS_COVERAGE}" == "1" ]; then
+  export MILVUS_CPP_COVERAGE_FILE=${TRAVIS_BUILD_DIR}/output_new.info
+fi
+
+export MILVUS_BUILD_TYPE=${MILVUS_BUILD_TYPE:=Release}

core/CMakeLists.txt

@@ -18,7 +18,7 @@
 #-------------------------------------------------------------------------------
 
 
-cmake_minimum_required(VERSION 3.14)
+cmake_minimum_required(VERSION 3.12)
 message(STATUS "Building using CMake version: ${CMAKE_VERSION}")
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
@@ -31,16 +31,14 @@ GET_CURRENT_TIME(BUILD_TIME)
 string(REGEX REPLACE "\n" "" BUILD_TIME ${BUILD_TIME})
 message(STATUS "Build time = ${BUILD_TIME}")
 
-MACRO (GET_GIT_BRANCH_NAME GIT_BRANCH_NAME)
-    execute_process(COMMAND "git" rev-parse --abbrev-ref HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
-    if(GIT_BRANCH_NAME STREQUAL "")
-        execute_process(COMMAND "git" symbolic-ref --short -q HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
-    endif()
-ENDMACRO (GET_GIT_BRANCH_NAME)
+MACRO(GET_GIT_BRANCH_NAME GIT_BRANCH_NAME)
+    execute_process(COMMAND sh "-c" "git log --decorate | head -n 1 | sed 's/.*(\\(.*\\))/\\1/' | sed 's/.* \\(.*\\),.*/\\1/' | sed 's=[a-zA-Z]*\/==g'"
+                    OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
+ENDMACRO(GET_GIT_BRANCH_NAME)
 
 GET_GIT_BRANCH_NAME(GIT_BRANCH_NAME)
 message(STATUS "GIT_BRANCH_NAME = ${GIT_BRANCH_NAME}")
-if(NOT GIT_BRANCH_NAME STREQUAL "")
+if (NOT GIT_BRANCH_NAME STREQUAL "")
     string(REGEX REPLACE "\n" "" GIT_BRANCH_NAME ${GIT_BRANCH_NAME})
 endif ()
 
@@ -69,7 +67,7 @@ if (MILVUS_VERSION_MAJOR STREQUAL ""
         OR MILVUS_VERSION_PATCH STREQUAL "")
     message(WARNING "Failed to determine Milvus version from git branch name")
     set(MILVUS_VERSION "0.6.0")
-endif()
+endif ()
 
 message(STATUS "Build version = ${MILVUS_VERSION}")
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/src/config.h.in ${CMAKE_CURRENT_SOURCE_DIR}/src/config.h @ONLY)
@@ -112,23 +110,36 @@ endif ()
 set(MILVUS_SOURCE_DIR ${PROJECT_SOURCE_DIR})
 set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR})
 set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src)
+set(MILVUS_THIRDPARTY_SRC ${PROJECT_SOURCE_DIR}/thirdparty)
 
 include(ExternalProject)
 include(DefineOptions)
 include(BuildUtils)
 include(ThirdPartyPackages)
 
-set(MILVUS_GPU_VERSION false)
-if (MILVUS_CPU_VERSION)
-    message(STATUS "Building Milvus CPU version")
-    add_compile_definitions("MILVUS_CPU_VERSION")
-else ()
+if(MILVUS_USE_CCACHE)
+  find_program(CCACHE_FOUND ccache)
+  if(CCACHE_FOUND)
+    message(STATUS "Using ccache: ${CCACHE_FOUND}")
+    set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND})
+    set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND})
+    # let ccache preserve C++ comments, because some of them may be
+    # meaningful to the compiler
+    set(ENV{CCACHE_COMMENTS} "1")
+  endif(CCACHE_FOUND)
+endif()
+
+set(MILVUS_CPU_VERSION false)
+if (MILVUS_GPU_VERSION)
     message(STATUS "Building Milvus GPU version")
-    set(MILVUS_GPU_VERSION true)
     add_compile_definitions("MILVUS_GPU_VERSION")
     enable_language(CUDA)
     find_package(CUDA 10 REQUIRED)
     set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES --expt-extended-lambda")
+else ()
+    message(STATUS "Building Milvus CPU version")
+    set(MILVUS_CPU_VERSION true)
+    add_compile_definitions("MILVUS_CPU_VERSION")
 endif ()
 
 if (CMAKE_BUILD_TYPE STREQUAL "Release")
@@ -162,7 +173,13 @@ add_custom_target(Clean-All COMMAND ${CMAKE_BUILD_TOOL} clean)
 if ("${MILVUS_DB_PATH}" STREQUAL "")
     set(MILVUS_DB_PATH "/tmp/milvus")
 endif ()
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml)
+
+if (MILVUS_GPU_VERSION)
+    configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_gpu_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml)
+else()
+    configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_cpu_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml)
+endif()
+
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.conf)
 
 install(DIRECTORY scripts/
@@ -273,4 +290,4 @@ if (${CLANG_TIDY_FOUND})
             --source_dir
             ${CMAKE_CURRENT_SOURCE_DIR}/src
             ${MILVUS_LINT_QUIET})
-endif ()
\ No newline at end of file
+endif ()

core/build-support/lint_exclusions.txt

@@ -6,5 +6,5 @@
 *easylogging++*
 *SqliteMetaImpl.cpp
 *src/grpc*
-*src/external*
+*thirdparty*
 *milvus/include*
\ No newline at end of file

core/build.sh

@@ -12,18 +12,12 @@ USE_JFROG_CACHE="OFF"
 RUN_CPPLINT="OFF"
 CUSTOMIZATION="OFF" # default use ori faiss
 CUDA_COMPILER=/usr/local/cuda/bin/nvcc
-CPU_VERSION="OFF"
+GPU_VERSION="OFF" #defaults to CPU version
 WITH_MKL="OFF"
+FAISS_ROOT=""
+FAISS_SOURCE="BUNDLED"
 
-CUSTOMIZED_FAISS_URL="${FAISS_URL:-NONE}"
-wget -q --method HEAD ${CUSTOMIZED_FAISS_URL}
-if [ $? -eq 0 ]; then
-  CUSTOMIZATION="ON"
-else
-  CUSTOMIZATION="OFF"
-fi
-
-while getopts "p:d:t:ulrcgjhxzm" arg
+while getopts "p:d:t:f:ulrcgjhxzm" arg
 do
         case $arg in
              p)
@@ -35,6 +29,10 @@ do
              t)
                 BUILD_TYPE=$OPTARG # BUILD_TYPE
                 ;;
+             f)
+                FAISS_ROOT=$OPTARG
+                FAISS_SOURCE="AUTO"
+                ;;
              u)
                 echo "Build and run unittest cases" ;
                 BUILD_UNITTEST="ON";
@@ -51,7 +49,7 @@ do
              c)
                 BUILD_COVERAGE="ON"
                 ;;
-             g)
+             z)
                 PROFILING="ON"
                 ;;
              j)
@@ -60,8 +58,8 @@ do
              x)
                 CUSTOMIZATION="OFF" # force use ori faiss
                 ;;
-             z)
-                CPU_VERSION="ON"
+             g)
+                GPU_VERSION="ON"
                 ;;
              m)
                 WITH_MKL="ON"
@@ -73,18 +71,19 @@ parameter:
 -p: install prefix(default: $(pwd)/milvus)
 -d: db data path(default: /tmp/milvus)
 -t: build type(default: Debug)
+-f: faiss root path(default: empty)
 -u: building unit test options(default: OFF)
 -l: run cpplint, clang-format and clang-tidy(default: OFF)
 -r: remove previous build directory(default: OFF)
 -c: code coverage(default: OFF)
--g: profiling(default: OFF)
+-z: profiling(default: OFF)
 -j: use jfrog cache build directory(default: OFF)
--z: build pure CPU version(default: OFF)
+-g: build GPU version(default: OFF)
 -m: build with MKL(default: OFF)
 -h: help
 
 usage:
-./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} [-u] [-l] [-r] [-c] [-g] [-j] [-z] [-m] [-h]
+./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -f \${FAISS_ROOT} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-h]
                 "
                 exit 0
                 ;;
@@ -109,15 +108,16 @@ CMAKE_CMD="cmake \
 -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
 -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}
 -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
+-DFAISS_ROOT=${FAISS_ROOT} \
+-DFAISS_SOURCE=${FAISS_SOURCE} \
 -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
 -DBUILD_COVERAGE=${BUILD_COVERAGE} \
 -DMILVUS_DB_PATH=${DB_PATH} \
 -DMILVUS_ENABLE_PROFILING=${PROFILING} \
 -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
 -DCUSTOMIZATION=${CUSTOMIZATION} \
--DFAISS_URL=${CUSTOMIZED_FAISS_URL} \
--DMILVUS_CPU_VERSION=${CPU_VERSION} \
--DBUILD_FAISS_WITH_MKL=${WITH_MKL} \
+-DMILVUS_GPU_VERSION=${GPU_VERSION} \
+-DFAISS_WITH_MKL=${WITH_MKL} \
 ../"
 echo ${CMAKE_CMD}
 ${CMAKE_CMD}

core/cmake/DefineOptions.cmake

@@ -13,16 +13,16 @@ macro(define_option name description default)
 endmacro()
 
 function(list_join lst glue out)
-    if("${${lst}}" STREQUAL "")
+    if ("${${lst}}" STREQUAL "")
         set(${out} "" PARENT_SCOPE)
         return()
-    endif()
+    endif ()
 
     list(GET ${lst} 0 joined)
     list(REMOVE_AT ${lst} 0)
-    foreach(item ${${lst}})
+    foreach (item ${${lst}})
         set(joined "${joined}${glue}${item}")
-    endforeach()
+    endforeach ()
     set(${out} ${joined} PARENT_SCOPE)
 endfunction()
 
@@ -35,27 +35,29 @@ macro(define_option_string name description default)
 
     set("${name}_OPTION_ENUM" ${ARGN})
     list_join("${name}_OPTION_ENUM" "|" "${name}_OPTION_ENUM")
-    if(NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
+    if (NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
         set_property(CACHE ${name} PROPERTY STRINGS ${ARGN})
-    endif()
+    endif ()
 endmacro()
 
 #----------------------------------------------------------------------
-set_option_category("CPU version")
+set_option_category("GPU version")
 
-define_option(MILVUS_CPU_VERSION "Build CPU version only" OFF)
+define_option(MILVUS_GPU_VERSION "Build GPU version" OFF)
 
 #----------------------------------------------------------------------
 set_option_category("Thirdparty")
 
-set(MILVUS_DEPENDENCY_SOURCE_DEFAULT "AUTO")
+set(MILVUS_DEPENDENCY_SOURCE_DEFAULT "BUNDLED")
 
 define_option_string(MILVUS_DEPENDENCY_SOURCE
-                    "Method to use for acquiring MILVUS's build dependencies"
-                    "${MILVUS_DEPENDENCY_SOURCE_DEFAULT}"
-                    "AUTO"
-                    "BUNDLED"
-                    "SYSTEM")
+        "Method to use for acquiring MILVUS's build dependencies"
+        "${MILVUS_DEPENDENCY_SOURCE_DEFAULT}"
+        "AUTO"
+        "BUNDLED"
+        "SYSTEM")
+
+define_option(MILVUS_USE_CCACHE "Use ccache when compiling (if available)" ON)
 
 define_option(MILVUS_VERBOSE_THIRDPARTY_BUILD
         "Show output from ExternalProjects rather than just logging to files" ON)
@@ -75,33 +77,21 @@ define_option(MILVUS_WITH_YAMLCPP "Build with yaml-cpp library" ON)
 if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
     define_option(MILVUS_WITH_LIBUNWIND "Build with libunwind" ON)
     define_option(MILVUS_WITH_GPERFTOOLS "Build with gperftools" ON)
-endif()
+endif ()
 
 define_option(MILVUS_WITH_GRPC "Build with GRPC" ON)
 
 define_option(MILVUS_WITH_ZLIB "Build with zlib compression" ON)
 
-#----------------------------------------------------------------------
-if(MSVC)
-    set_option_category("MSVC")
-
-    define_option(MSVC_LINK_VERBOSE
-            "Pass verbose linking options when linking libraries and executables"
-            OFF)
-
-    define_option(MILVUS_USE_STATIC_CRT "Build MILVUS with statically linked CRT" OFF)
-endif()
-
-
 #----------------------------------------------------------------------
 set_option_category("Test and benchmark")
 
 unset(MILVUS_BUILD_TESTS CACHE)
 if (BUILD_UNIT_TEST)
     define_option(MILVUS_BUILD_TESTS "Build the MILVUS googletest unit tests" ON)
-else()
+else ()
     define_option(MILVUS_BUILD_TESTS "Build the MILVUS googletest unit tests" OFF)
-endif(BUILD_UNIT_TEST)
+endif (BUILD_UNIT_TEST)
 
 #----------------------------------------------------------------------
 macro(config_summary)
@@ -113,12 +103,12 @@ macro(config_summary)
     message(STATUS "  Generator: ${CMAKE_GENERATOR}")
     message(STATUS "  Build type: ${CMAKE_BUILD_TYPE}")
     message(STATUS "  Source directory: ${CMAKE_CURRENT_SOURCE_DIR}")
-    if(${CMAKE_EXPORT_COMPILE_COMMANDS})
+    if (${CMAKE_EXPORT_COMPILE_COMMANDS})
         message(
                 STATUS "  Compile commands: ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json")
-    endif()
+    endif ()
 
-    foreach(category ${MILVUS_OPTION_CATEGORIES})
+    foreach (category ${MILVUS_OPTION_CATEGORIES})
 
         message(STATUS)
         message(STATUS "${category} options:")
@@ -126,50 +116,50 @@ macro(config_summary)
         set(option_names ${MILVUS_${category}_OPTION_NAMES})
 
         set(max_value_length 0)
-        foreach(name ${option_names})
+        foreach (name ${option_names})
             string(LENGTH "\"${${name}}\"" value_length)
-            if(${max_value_length} LESS ${value_length})
+            if (${max_value_length} LESS ${value_length})
                 set(max_value_length ${value_length})
-            endif()
-        endforeach()
+            endif ()
+        endforeach ()
 
-        foreach(name ${option_names})
-            if("${${name}_OPTION_TYPE}" STREQUAL "string")
+        foreach (name ${option_names})
+            if ("${${name}_OPTION_TYPE}" STREQUAL "string")
                 set(value "\"${${name}}\"")
-            else()
+            else ()
                 set(value "${${name}}")
-            endif()
+            endif ()
 
             set(default ${${name}_OPTION_DEFAULT})
             set(description ${${name}_OPTION_DESCRIPTION})
             string(LENGTH ${description} description_length)
-            if(${description_length} LESS 70)
+            if (${description_length} LESS 70)
                 string(
                         SUBSTRING
                         "                                                                     "
                         ${description_length} -1 description_padding)
-            else()
+            else ()
                 set(description_padding "
                 ")
-            endif()
+            endif ()
 
             set(comment "[${name}]")
 
-            if("${value}" STREQUAL "${default}")
+            if ("${value}" STREQUAL "${default}")
                 set(comment "[default] ${comment}")
-            endif()
+            endif ()
 
-            if(NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
+            if (NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
                 set(comment "${comment} [${${name}_OPTION_ENUM}]")
-            endif()
+            endif ()
 
             string(
                     SUBSTRING "${value}                                                             "
                     0 ${max_value_length} value)
 
             message(STATUS "  ${description} ${description_padding} ${value} ${comment}")
-        endforeach()
+        endforeach ()
 
-    endforeach()
+    endforeach ()
 
 endmacro()

core/cmake/ThirdPartyPackages.cmake

@@ -164,8 +164,10 @@ endif ()
 
 macro(resolve_dependency DEPENDENCY_NAME)
     if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO")
-        #disable find_package for now
-        build_dependency(${DEPENDENCY_NAME})
+        find_package(${DEPENDENCY_NAME} MODULE)
+        if(NOT ${${DEPENDENCY_NAME}_FOUND})
+          build_dependency(${DEPENDENCY_NAME})
+        endif()
     elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED")
         build_dependency(${DEPENDENCY_NAME})
     elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM")

core/conf/server_cpu_config.template

+# Default values are used when you make no changes to the following parameters.
+
+server_config:
+  address: 0.0.0.0                  # milvus server ip address (IPv4)
+  port: 19530                       # milvus server port, must in range [1025, 65534]
+  deploy_mode: single               # deployment type: single, cluster_readonly, cluster_writable
+  time_zone: UTC+8                  # time zone, must be in format: UTC+X
+
+db_config:
+  primary_path: @MILVUS_DB_PATH@    # path used to store data and meta
+  secondary_path:                   # path used to store data only, split by semicolon
+
+  backend_url: sqlite://:@:/        # URI format: dialect://username:password@host:port/database
+                                    # Keep 'dialect://:@:/', and replace other texts with real values
+                                    # Replace 'dialect' with 'mysql' or 'sqlite'
+
+  insert_buffer_size: 4             # GB, maximum insert buffer size allowed, must be a positive integer
+                                    # sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory
+
+  preload_table:                    # preload data at startup, '*' means load all tables, empty value means no preload
+                                    # you can specify preload tables like this: table1,table2,table3
+
+metric_config:
+  enable_monitor: false             # enable monitoring or not, must be a boolean
+  collector: prometheus             # prometheus
+  prometheus_config:
+    port: 8080                      # port prometheus uses to fetch metrics, must in range [1025, 65534]
+
+cache_config:
+
+  cpu_cache_capacity: 16            # GB, CPU memory used for cache, must be a positive integer
+  cpu_cache_threshold: 0.85         # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0]
+  cache_insert_data: false          # whether to load inserted data into cache, must be a boolean
+
+engine_config:
+  use_blas_threshold: 1100          # if nq <  use_blas_threshold, use SSE, faster with fluctuated response times
+                                    # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times
+  gpu_search_threshold: 1000        # threshold beyond which the search computation is executed on GPUs only
+
+resource_config:
+  search_resources:                 # define the device used for search computation
+    - cpu
+  index_build_device: cpu           # CPU used for building index

core/conf/server_config.template → core/conf/server_gpu_config.template

@@ -27,14 +27,11 @@ metric_config:
     port: 8080                      # port prometheus uses to fetch metrics, must in range [1025, 65534]
 
 cache_config:
-
   cpu_cache_capacity: 16            # GB, CPU memory used for cache, must be a positive integer
   cpu_cache_threshold: 0.85         # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0]
-  cache_insert_data: false          # whether to load inserted data into cache, must be a boolean
-
-# Skip the following config if you are using GPU version
-  gpu_cache_capacity: 4             # GB, GPU memory used for cache, must be a positive integer
+  gpu_cache_capacity: 3             # GB, GPU memory used for cache, must be a positive integer
   gpu_cache_threshold: 0.85         # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0]
+  cache_insert_data: false          # whether to load inserted data into cache, must be a boolean
 
 engine_config:
   use_blas_threshold: 1100          # if nq <  use_blas_threshold, use SSE, faster with fluctuated response times
@@ -45,4 +42,4 @@ resource_config:
   search_resources:                 # define the devices used for search computation, must be in format: cpu or gpux
     - cpu
     - gpu0
-  index_build_device: gpu0           # CPU / GPU used for building index, must be in format: cpu / gpux
+  index_build_device: gpu0          # CPU / GPU used for building index, must be in format: cpu or gpux

core/coverage.sh

@@ -122,9 +122,7 @@ ${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \
     "*/src/server/Server.cpp" \
     "*/src/server/DBWrapper.cpp" \
     "*/src/server/grpc_impl/GrpcServer.cpp" \
-    "*/easylogging++.h" \
-    "*/easylogging++.cc" \
-    "*/src/external/*"
+    "*/thirdparty/*"
 
 if [ $? -ne 0 ]; then
     echo "generate ${FILE_INFO_OUTPUT_NEW} failed"

core/migration/README.md

+## Data Migration
+
+####0.3.x
+legacy data is not migrate-able for later versions
+
+####0.4.x
+legacy data can be reused directly by 0.5.x
+
+legacy data can be migrated to 0.6.x
+
+####0.5.x
+legacy data can be migrated to 0.6.x
+
+####0.6.x
+how to migrate legacy 0.4.x/0.5.x data
+
+for sqlite meta:
+```shell
+ $ sqlite3 [parth_to]/meta.sqlite < sqlite_4_to_6.sql
+```
+
+for mysql meta:
+```shell
+ $ mysql -h127.0.0.1 -uroot -p123456 -Dmilvus < mysql_4_to_6.sql
+```
+
+
+

core/migration/mysql_4_to_6.sql

+alter table Tables add column owner_table VARCHAR(255) DEFAULT '' NOT NULL;
+alter table Tables add column partition_tag VARCHAR(255) DEFAULT '' NOT NULL;
+alter table Tables add column version VARCHAR(64) DEFAULT '0.6.0' NOT NULL;
+update Tables set version='0.6.0';

core/migration/sqlite_4_to_6.sql

+alter table Tables add column 'owner_table' TEXT DEFAULT '' NOT NULL;
+alter table Tables add column 'partition_tag' TEXT DEFAULT '' NOT NULL;
+alter table Tables add column 'version' TEXT DEFAULT '0.6.0' NOT NULL;
+update Tables set version='0.6.0';

core/src/CMakeLists.txt

@@ -19,11 +19,15 @@
 
 include_directories(${MILVUS_SOURCE_DIR})
 include_directories(${MILVUS_ENGINE_SRC})
+include_directories(${MILVUS_THIRDPARTY_SRC})
 
 include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-status)
 include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-milvus)
 
 add_subdirectory(index)
+if (FAISS_WITH_MKL)
+    add_compile_definitions("WITH_MKL")
+endif ()
 
 set(INDEX_INCLUDE_DIRS ${INDEX_INCLUDE_DIRS} PARENT_SCOPE)
 foreach (dir ${INDEX_INCLUDE_DIRS})
@@ -62,11 +66,11 @@ set(scheduler_files
         ${scheduler_task_files}
         )
 
-aux_source_directory(${MILVUS_ENGINE_SRC}/external/easyloggingpp external_easyloggingpp_files)
-aux_source_directory(${MILVUS_ENGINE_SRC}/external/nlohmann external_nlohmann_files)
-set(external_files
-        ${external_easyloggingpp_files}
-        ${external_nlohmann_files}
+aux_source_directory(${MILVUS_THIRDPARTY_SRC}/easyloggingpp thirdparty_easyloggingpp_files)
+aux_source_directory(${MILVUS_THIRDPARTY_SRC}/nlohmann thirdparty_nlohmann_files)
+set(thirdparty_files
+        ${thirdparty_easyloggingpp_files}
+        ${thirdparty_nlohmann_files}
         )
 
 aux_source_directory(${MILVUS_ENGINE_SRC}/server server_files)
@@ -82,7 +86,7 @@ set(engine_files
         ${db_insert_files}
         ${db_meta_files}
         ${metrics_files}
-        ${external_files}
+        ${thirdparty_files}
         ${utils_files}
         ${wrapper_files}
         )

core/src/db/DB.h

@@ -47,43 +47,68 @@ class DB {
 
     virtual Status
     CreateTable(meta::TableSchema& table_schema_) = 0;
+
     virtual Status
-    DeleteTable(const std::string& table_id, const meta::DatesT& dates) = 0;
+    DropTable(const std::string& table_id, const meta::DatesT& dates) = 0;
+
     virtual Status
     DescribeTable(meta::TableSchema& table_schema_) = 0;
+
     virtual Status
     HasTable(const std::string& table_id, bool& has_or_not_) = 0;
+
     virtual Status
     AllTables(std::vector<meta::TableSchema>& table_schema_array) = 0;
+
     virtual Status
     GetTableRowCount(const std::string& table_id, uint64_t& row_count) = 0;
+
     virtual Status
     PreloadTable(const std::string& table_id) = 0;
+
     virtual Status
     UpdateTableFlag(const std::string& table_id, int64_t flag) = 0;
 
     virtual Status
-    InsertVectors(const std::string& table_id_, uint64_t n, const float* vectors, IDNumbers& vector_ids_) = 0;
+    CreatePartition(const std::string& table_id, const std::string& partition_name,
+                    const std::string& partition_tag) = 0;
+
+    virtual Status
+    DropPartition(const std::string& partition_name) = 0;
 
     virtual Status
-    Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
-          QueryResults& results) = 0;
+    DropPartitionByTag(const std::string& table_id, const std::string& partition_tag) = 0;
 
     virtual Status
-    Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
-          const meta::DatesT& dates, QueryResults& results) = 0;
+    ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partiton_schema_array) = 0;
 
     virtual Status
-    Query(const std::string& table_id, const std::vector<std::string>& file_ids, uint64_t k, uint64_t nq,
-          uint64_t nprobe, const float* vectors, const meta::DatesT& dates, QueryResults& results) = 0;
+    InsertVectors(const std::string& table_id, const std::string& partition_tag, uint64_t n, const float* vectors,
+                  IDNumbers& vector_ids_) = 0;
+
+    virtual Status
+    Query(const std::string& table_id, const std::vector<std::string>& partition_tags, uint64_t k, uint64_t nq,
+          uint64_t nprobe, const float* vectors, ResultIds& result_ids, ResultDistances& result_distances) = 0;
+
+    virtual Status
+    Query(const std::string& table_id, const std::vector<std::string>& partition_tags, uint64_t k, uint64_t nq,
+          uint64_t nprobe, const float* vectors, const meta::DatesT& dates, ResultIds& result_ids,
+          ResultDistances& result_distances) = 0;
+
+    virtual Status
+    QueryByFileID(const std::string& table_id, const std::vector<std::string>& file_ids, uint64_t k, uint64_t nq,
+                  uint64_t nprobe, const float* vectors, const meta::DatesT& dates, ResultIds& result_ids,
+                  ResultDistances& result_distances) = 0;
 
     virtual Status
     Size(uint64_t& result) = 0;
 
     virtual Status
     CreateIndex(const std::string& table_id, const TableIndex& index) = 0;
+
     virtual Status
     DescribeIndex(const std::string& table_id, TableIndex& index) = 0;
+
     virtual Status
     DropIndex(const std::string& table_id) = 0;
 

core/src/db/DBImpl.h

@@ -57,7 +57,7 @@ class DBImpl : public DB {
     CreateTable(meta::TableSchema& table_schema) override;
 
     Status
-    DeleteTable(const std::string& table_id, const meta::DatesT& dates) override;
+    DropTable(const std::string& table_id, const meta::DatesT& dates) override;
 
     Status
     DescribeTable(meta::TableSchema& table_schema) override;
@@ -78,7 +78,21 @@ class DBImpl : public DB {
     GetTableRowCount(const std::string& table_id, uint64_t& row_count) override;
 
     Status
-    InsertVectors(const std::string& table_id, uint64_t n, const float* vectors, IDNumbers& vector_ids) override;
+    CreatePartition(const std::string& table_id, const std::string& partition_name,
+                    const std::string& partition_tag) override;
+
+    Status
+    DropPartition(const std::string& partition_name) override;
+
+    Status
+    DropPartitionByTag(const std::string& table_id, const std::string& partition_tag) override;
+
+    Status
+    ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partiton_schema_array) override;
+
+    Status
+    InsertVectors(const std::string& table_id, const std::string& partition_tag, uint64_t n, const float* vectors,
+                  IDNumbers& vector_ids) override;
 
     Status
     CreateIndex(const std::string& table_id, const TableIndex& index) override;
@@ -90,16 +104,18 @@ class DBImpl : public DB {
     DropIndex(const std::string& table_id) override;
 
     Status
-    Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
-          QueryResults& results) override;
+    Query(const std::string& table_id, const std::vector<std::string>& partition_tags, uint64_t k, uint64_t nq,
+          uint64_t nprobe, const float* vectors, ResultIds& result_ids, ResultDistances& result_distances) override;
 
     Status
-    Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
-          const meta::DatesT& dates, QueryResults& results) override;
+    Query(const std::string& table_id, const std::vector<std::string>& partition_tags, uint64_t k, uint64_t nq,
+          uint64_t nprobe, const float* vectors, const meta::DatesT& dates, ResultIds& result_ids,
+          ResultDistances& result_distances) override;
 
     Status
-    Query(const std::string& table_id, const std::vector<std::string>& file_ids, uint64_t k, uint64_t nq,
-          uint64_t nprobe, const float* vectors, const meta::DatesT& dates, QueryResults& results) override;
+    QueryByFileID(const std::string& table_id, const std::vector<std::string>& file_ids, uint64_t k, uint64_t nq,
+                  uint64_t nprobe, const float* vectors, const meta::DatesT& dates, ResultIds& result_ids,
+                  ResultDistances& result_distances) override;
 
     Status
     Size(uint64_t& result) override;
@@ -107,7 +123,7 @@ class DBImpl : public DB {
  private:
     Status
     QueryAsync(const std::string& table_id, const meta::TableFilesSchema& files, uint64_t k, uint64_t nq,
-               uint64_t nprobe, const float* vectors, QueryResults& results);
+               uint64_t nprobe, const float* vectors, ResultIds& result_ids, ResultDistances& result_distances);
 
     void
     BackgroundTimerTask();
@@ -136,6 +152,28 @@ class DBImpl : public DB {
     Status
     MemSerialize();
 
+    Status
+    GetFilesToSearch(const std::string& table_id, const std::vector<size_t>& file_ids, meta::TableFilesSchema& files);
+
+    Status
+    GetPartitionsByTags(const std::string& table_id, const std::vector<std::string>& partition_tags,
+                        std::set<std::string>& partition_name_array);
+
+    Status
+    DropTableRecursively(const std::string& table_id, const meta::DatesT& dates);
+
+    Status
+    UpdateTableIndexRecursively(const std::string& table_id, const TableIndex& index);
+
+    Status
+    BuildTableIndexRecursively(const std::string& table_id, const TableIndex& index);
+
+    Status
+    DropTableIndexRecursively(const std::string& table_id);
+
+    Status
+    GetTableRowCountRecursively(const std::string& table_id, uint64_t& row_count);
+
  private:
     const DBOptions options_;
 

core/src/db/Types.h

@@ -19,6 +19,7 @@
 
 #include "db/engine/ExecutionEngine.h"
 
+#include <faiss/Index.h>
 #include <stdint.h>
 #include <utility>
 #include <vector>
@@ -30,8 +31,8 @@ typedef int64_t IDNumber;
 typedef IDNumber* IDNumberPtr;
 typedef std::vector<IDNumber> IDNumbers;
 
-typedef std::vector<std::pair<IDNumber, double>> QueryResult;
-typedef std::vector<QueryResult> QueryResults;
+typedef std::vector<faiss::Index::idx_t> ResultIds;
+typedef std::vector<faiss::Index::distance_t> ResultDistances;
 
 struct TableIndex {
     int32_t engine_type_ = (int)EngineType::FAISS_IDMAP;

core/src/db/engine/ExecutionEngineImpl.cpp

@@ -117,7 +117,11 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
             break;
         }
         case EngineType::FAISS_PQ: {
+#ifdef MILVUS_CPU_VERSION
+            index = GetVecIndexFactory(IndexType::FAISS_IVFPQ_CPU);
+#else
             index = GetVecIndexFactory(IndexType::FAISS_IVFPQ_MIX);
+#endif
             break;
         }
         default: {

core/src/db/meta/Meta.h

@@ -50,14 +50,11 @@ class Meta {
     virtual Status
     AllTables(std::vector<TableSchema>& table_schema_array) = 0;
 
-    virtual Status
-    UpdateTableIndex(const std::string& table_id, const TableIndex& index) = 0;
-
     virtual Status
     UpdateTableFlag(const std::string& table_id, int64_t flag) = 0;
 
     virtual Status
-    DeleteTable(const std::string& table_id) = 0;
+    DropTable(const std::string& table_id) = 0;
 
     virtual Status
     DeleteTableFiles(const std::string& table_id) = 0;
@@ -66,32 +63,47 @@ class Meta {
     CreateTableFile(TableFileSchema& file_schema) = 0;
 
     virtual Status
-    DropPartitionsByDates(const std::string& table_id, const DatesT& dates) = 0;
+    DropDataByDate(const std::string& table_id, const DatesT& dates) = 0;
 
     virtual Status
     GetTableFiles(const std::string& table_id, const std::vector<size_t>& ids, TableFilesSchema& table_files) = 0;
 
+    virtual Status
+    UpdateTableFile(TableFileSchema& file_schema) = 0;
+
+    virtual Status
+    UpdateTableFiles(TableFilesSchema& files) = 0;
+
+    virtual Status
+    UpdateTableIndex(const std::string& table_id, const TableIndex& index) = 0;
+
     virtual Status
     UpdateTableFilesToIndex(const std::string& table_id) = 0;
 
     virtual Status
-    UpdateTableFile(TableFileSchema& file_schema) = 0;
+    DescribeTableIndex(const std::string& table_id, TableIndex& index) = 0;
 
     virtual Status
-    UpdateTableFiles(TableFilesSchema& files) = 0;
+    DropTableIndex(const std::string& table_id) = 0;
 
     virtual Status
-    FilesToSearch(const std::string& table_id, const std::vector<size_t>& ids, const DatesT& dates,
-                  DatePartionedTableFilesSchema& files) = 0;
+    CreatePartition(const std::string& table_name, const std::string& partition_name, const std::string& tag) = 0;
 
     virtual Status
-    FilesToMerge(const std::string& table_id, DatePartionedTableFilesSchema& files) = 0;
+    DropPartition(const std::string& partition_name) = 0;
 
     virtual Status
-    Size(uint64_t& result) = 0;
+    ShowPartitions(const std::string& table_name, std::vector<meta::TableSchema>& partiton_schema_array) = 0;
 
     virtual Status
-    Archive() = 0;
+    GetPartitionName(const std::string& table_name, const std::string& tag, std::string& partition_name) = 0;
+
+    virtual Status
+    FilesToSearch(const std::string& table_id, const std::vector<size_t>& ids, const DatesT& dates,
+                  DatePartionedTableFilesSchema& files) = 0;
+
+    virtual Status
+    FilesToMerge(const std::string& table_id, DatePartionedTableFilesSchema& files) = 0;
 
     virtual Status
     FilesToIndex(TableFilesSchema&) = 0;
@@ -101,10 +113,10 @@ class Meta {
                 std::vector<std::string>& file_ids) = 0;
 
     virtual Status
-    DescribeTableIndex(const std::string& table_id, TableIndex& index) = 0;
+    Size(uint64_t& result) = 0;
 
     virtual Status
-    DropTableIndex(const std::string& table_id) = 0;
+    Archive() = 0;
 
     virtual Status
     CleanUp() = 0;

core/src/db/meta/MetaTypes.h

@@ -19,6 +19,7 @@
 
 #include "db/Constants.h"
 #include "db/engine/ExecutionEngine.h"
+#include "src/config.h"
 
 #include <map>
 #include <memory>
@@ -33,6 +34,7 @@ constexpr int32_t DEFAULT_ENGINE_TYPE = (int)EngineType::FAISS_IDMAP;
 constexpr int32_t DEFAULT_NLIST = 16384;
 constexpr int32_t DEFAULT_METRIC_TYPE = (int)MetricType::L2;
 constexpr int32_t DEFAULT_INDEX_FILE_SIZE = ONE_GB;
+constexpr char CURRENT_VERSION[] = MILVUS_VERSION;
 
 constexpr int64_t FLAG_MASK_NO_USERID = 0x1;
 constexpr int64_t FLAG_MASK_HAS_USERID = 0x1 << 1;
@@ -57,6 +59,9 @@ struct TableSchema {
     int32_t engine_type_ = DEFAULT_ENGINE_TYPE;
     int32_t nlist_ = DEFAULT_NLIST;
     int32_t metric_type_ = DEFAULT_METRIC_TYPE;
+    std::string owner_table_;
+    std::string partition_tag_;
+    std::string version_ = CURRENT_VERSION;
 };  // TableSchema
 
 struct TableFileSchema {

core/src/db/meta/MySQLMetaImpl.h

@@ -49,7 +49,7 @@ class MySQLMetaImpl : public Meta {
     AllTables(std::vector<TableSchema>& table_schema_array) override;
 
     Status
-    DeleteTable(const std::string& table_id) override;
+    DropTable(const std::string& table_id) override;
 
     Status
     DeleteTableFiles(const std::string& table_id) override;
@@ -58,21 +58,26 @@ class MySQLMetaImpl : public Meta {
     CreateTableFile(TableFileSchema& file_schema) override;
 
     Status
-    DropPartitionsByDates(const std::string& table_id, const DatesT& dates) override;
+    DropDataByDate(const std::string& table_id, const DatesT& dates) override;
 
     Status
     GetTableFiles(const std::string& table_id, const std::vector<size_t>& ids, TableFilesSchema& table_files) override;
 
-    Status
-    FilesByType(const std::string& table_id, const std::vector<int>& file_types,
-                std::vector<std::string>& file_ids) override;
-
     Status
     UpdateTableIndex(const std::string& table_id, const TableIndex& index) override;
 
     Status
     UpdateTableFlag(const std::string& table_id, int64_t flag) override;
 
+    Status
+    UpdateTableFile(TableFileSchema& file_schema) override;
+
+    Status
+    UpdateTableFilesToIndex(const std::string& table_id) override;
+
+    Status
+    UpdateTableFiles(TableFilesSchema& files) override;
+
     Status
     DescribeTableIndex(const std::string& table_id, TableIndex& index) override;
 
@@ -80,13 +85,16 @@ class MySQLMetaImpl : public Meta {
     DropTableIndex(const std::string& table_id) override;
 
     Status
-    UpdateTableFile(TableFileSchema& file_schema) override;
+    CreatePartition(const std::string& table_id, const std::string& partition_name, const std::string& tag) override;
 
     Status
-    UpdateTableFilesToIndex(const std::string& table_id) override;
+    DropPartition(const std::string& partition_name) override;
 
     Status
-    UpdateTableFiles(TableFilesSchema& files) override;
+    ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partiton_schema_array) override;
+
+    Status
+    GetPartitionName(const std::string& table_id, const std::string& tag, std::string& partition_name) override;
 
     Status
     FilesToSearch(const std::string& table_id, const std::vector<size_t>& ids, const DatesT& dates,
@@ -98,6 +106,10 @@ class MySQLMetaImpl : public Meta {
     Status
     FilesToIndex(TableFilesSchema&) override;
 
+    Status
+    FilesByType(const std::string& table_id, const std::vector<int>& file_types,
+                std::vector<std::string>& file_ids) override;
+
     Status
     Archive() override;
 

core/src/db/meta/SqliteMetaImpl.h

@@ -49,7 +49,7 @@ class SqliteMetaImpl : public Meta {
     AllTables(std::vector<TableSchema>& table_schema_array) override;
 
     Status
-    DeleteTable(const std::string& table_id) override;
+    DropTable(const std::string& table_id) override;
 
     Status
     DeleteTableFiles(const std::string& table_id) override;
@@ -58,21 +58,26 @@ class SqliteMetaImpl : public Meta {
     CreateTableFile(TableFileSchema& file_schema) override;
 
     Status
-    DropPartitionsByDates(const std::string& table_id, const DatesT& dates) override;
+    DropDataByDate(const std::string& table_id, const DatesT& dates) override;
 
     Status
     GetTableFiles(const std::string& table_id, const std::vector<size_t>& ids, TableFilesSchema& table_files) override;
 
-    Status
-    FilesByType(const std::string& table_id, const std::vector<int>& file_types,
-                std::vector<std::string>& file_ids) override;
-
     Status
     UpdateTableIndex(const std::string& table_id, const TableIndex& index) override;
 
     Status
     UpdateTableFlag(const std::string& table_id, int64_t flag) override;
 
+    Status
+    UpdateTableFile(TableFileSchema& file_schema) override;
+
+    Status
+    UpdateTableFilesToIndex(const std::string& table_id) override;
+
+    Status
+    UpdateTableFiles(TableFilesSchema& files) override;
+
     Status
     DescribeTableIndex(const std::string& table_id, TableIndex& index) override;
 
@@ -80,13 +85,16 @@ class SqliteMetaImpl : public Meta {
     DropTableIndex(const std::string& table_id) override;
 
     Status
-    UpdateTableFilesToIndex(const std::string& table_id) override;
+    CreatePartition(const std::string& table_id, const std::string& partition_name, const std::string& tag) override;
 
     Status
-    UpdateTableFile(TableFileSchema& file_schema) override;
+    DropPartition(const std::string& partition_name) override;
 
     Status
-    UpdateTableFiles(TableFilesSchema& files) override;
+    ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partiton_schema_array) override;
+
+    Status
+    GetPartitionName(const std::string& table_id, const std::string& tag, std::string& partition_name) override;
 
     Status
     FilesToSearch(const std::string& table_id, const std::vector<size_t>& ids, const DatesT& dates,
@@ -99,11 +107,15 @@ class SqliteMetaImpl : public Meta {
     FilesToIndex(TableFilesSchema&) override;
 
     Status
-    Archive() override;
+    FilesByType(const std::string& table_id, const std::vector<int>& file_types,
+                std::vector<std::string>& file_ids) override;
 
     Status
     Size(uint64_t& result) override;
 
+    Status
+    Archive() override;
+
     Status
     CleanUp() override;
 

core/src/grpc/milvus.proto

@@ -5,14 +5,21 @@ import "status.proto";
 package milvus.grpc;
 
 /**
- * @brief Table Name
+ * @brief Table name
  */
 message TableName {
     string table_name = 1;
 }
 
 /**
- * @brief Table Name List
+ * @brief Partition name
+ */
+message PartitionName {
+    string partition_name = 1;
+}
+
+/**
+ * @brief Table name list
  */
 message TableNameList {
     Status status = 1;
@@ -20,7 +27,7 @@ message TableNameList {
 }
 
 /**
- * @brief Table Schema
+ * @brief Table schema
  */
 message TableSchema {
     Status status = 1;
@@ -31,7 +38,24 @@ message TableSchema {
 }
 
 /**
- * @brief Range Schema
+ * @brief Params of partition
+ */
+message PartitionParam {
+    string table_name = 1;
+    string partition_name = 2;
+    string tag = 3;
+}
+
+/**
+ * @brief Partition list
+ */
+message PartitionList {
+    Status status = 1;
+    repeated PartitionParam partition_array = 2;
+}
+
+/**
+ * @brief Range schema
  */
 message Range {
     string start_value = 1;
@@ -46,12 +70,13 @@ message RowRecord {
 }
 
 /**
- * @brief params to be inserted
+ * @brief Params to be inserted
  */
 message InsertParam {
     string table_name = 1;
     repeated RowRecord row_record_array = 2;
     repeated int64 row_id_array = 3;            //optional
+    string partition_tag = 4;
 }
 
 /**
@@ -63,7 +88,7 @@ message VectorIds {
 }
 
 /**
- * @brief params for searching vector
+ * @brief Params for searching vector
  */
 message SearchParam {
     string table_name = 1;
@@ -71,10 +96,11 @@ message SearchParam {
     repeated Range query_range_array = 3;
     int64 topk = 4;
     int64 nprobe = 5;
+    repeated string partition_tag_array = 6;
 }
 
 /**
- * @brief params for searching vector in files
+ * @brief Params for searching vector in files
  */
 message SearchInFilesParam {
     repeated string file_id_array = 1;
@@ -105,7 +131,7 @@ message TopKQueryResultList {
 }
 
 /**
- * @brief Server String Reply
+ * @brief Server string Reply
  */
 message StringReply {
     Status status = 1;
@@ -129,7 +155,7 @@ message TableRowCount {
 }
 
 /**
- * @brief Give Server Command
+ * @brief Give server Command
  */
 message Command {
     string cmd = 1;
@@ -155,169 +181,173 @@ message IndexParam {
 }
 
 /**
- * @brief table name and range for DeleteByRange
+ * @brief table name and range for DeleteByDate
  */
-message DeleteByRangeParam {
+message DeleteByDateParam {
     Range range = 1;
     string table_name = 2;
 }
 
 service MilvusService {
     /**
-     * @brief Create table method
-     *
-     * This method is used to create table
+     * @brief This method is used to create table
      *
-     * @param param, use to provide table information to be created.
+     * @param TableSchema, use to provide table information to be created.
      *
+     * @return Status
      */
     rpc CreateTable(TableSchema) returns (Status){}
 
     /**
-     * @brief Test table existence method
+     * @brief This method is used to test table existence.
      *
-     * This method is used to test table existence.
-     *
-     * @param table_name, table name is going to be tested.
+     * @param TableName, table name is going to be tested.
      *
+     * @return BoolReply
      */
     rpc HasTable(TableName) returns (BoolReply) {}
 
     /**
-     * @brief Delete table method
-     *
-     * This method is used to delete table.
+     * @brief This method is used to get table schema.
      *
-     * @param table_name, table name is going to be deleted.
+     * @param TableName, target table name.
      *
+     * @return TableSchema
      */
-    rpc DropTable(TableName) returns (Status) {}
+    rpc DescribeTable(TableName) returns (TableSchema) {}
 
     /**
-     * @brief Build index by table method
+     * @brief This method is used to get table schema.
      *
-     * This method is used to build index by table in sync mode.
-     *
-     * @param table_name, table is going to be built index.
+     * @param TableName, target table name.
      *
+     * @return TableRowCount
      */
-    rpc CreateIndex(IndexParam) returns (Status) {}
+    rpc CountTable(TableName) returns (TableRowCount) {}
 
     /**
-     * @brief Add vector array to table
-     *
-     * This method is used to add vector array to table.
+     * @brief This method is used to list all tables.
      *
-     * @param table_name, table_name is inserted.
-     * @param record_array, vector array is inserted.
+     * @param Command, dummy parameter.
      *
-     * @return vector id array
+     * @return TableNameList
      */
-    rpc Insert(InsertParam) returns (VectorIds) {}
+    rpc ShowTables(Command) returns (TableNameList) {}
 
     /**
-     * @brief Query vector
+     * @brief This method is used to delete table.
      *
-     * This method is used to query vector in table.
+     * @param TableName, table name is going to be deleted.
      *
-     * @param table_name, table_name is queried.
-     * @param query_record_array, all vector are going to be queried.
-     * @param query_range_array, optional ranges for conditional search. If not specified, search whole table
-     * @param topk, how many similarity vectors will be searched.
+     * @return TableNameList
+     */
+    rpc DropTable(TableName) returns (Status) {}
+
+    /**
+     * @brief This method is used to build index by table in sync mode.
      *
-     * @return query result array.
+     * @param IndexParam, index paramters.
+     *
+     * @return Status
      */
-    rpc Search(SearchParam) returns (TopKQueryResultList) {}
+    rpc CreateIndex(IndexParam) returns (Status) {}
 
     /**
-     * @brief Internal use query interface
+     * @brief This method is used to describe index
+     *
+     * @param TableName, target table name.
      *
-     * This method is used to query vector in specified files.
+     * @return IndexParam
+     */
+    rpc DescribeIndex(TableName) returns (IndexParam) {}
+
+    /**
+     * @brief This method is used to drop index
      *
-     * @param file_id_array, specified files id array, queried.
-     * @param query_record_array, all vector are going to be queried.
-     * @param query_range_array, optional ranges for conditional search. If not specified, search whole table
-     * @param topk, how many similarity vectors will be searched.
+     * @param TableName, target table name.
      *
-     * @return query result array.
+     * @return Status
      */
-    rpc SearchInFiles(SearchInFilesParam) returns (TopKQueryResultList) {}
+    rpc DropIndex(TableName) returns (Status) {}
 
     /**
-     * @brief Get table schema
+     * @brief This method is used to create partition
      *
-     * This method is used to get table schema.
+     * @param PartitionParam, partition parameters.
      *
-     * @param table_name, target table name.
+     * @return Status
+     */
+    rpc CreatePartition(PartitionParam) returns (Status) {}
+
+    /**
+     * @brief This method is used to show partition information
      *
-     * @return table schema
+     * @param TableName, target table name.
+     *
+     * @return PartitionList
      */
-    rpc DescribeTable(TableName) returns (TableSchema) {}
+    rpc ShowPartitions(TableName) returns (PartitionList) {}
 
     /**
-     * @brief Get table schema
+     * @brief This method is used to drop partition
      *
-     * This method is used to get table schema.
+     * @param PartitionParam, target partition.
      *
-     * @param table_name, target table name.
+     * @return Status
+     */
+    rpc DropPartition(PartitionParam) returns (Status) {}
+
+    /**
+     * @brief This method is used to add vector array to table.
+     *
+     * @param InsertParam, insert parameters.
      *
-     * @return table schema
+     * @return VectorIds
      */
-    rpc CountTable(TableName) returns (TableRowCount) {}
+    rpc Insert(InsertParam) returns (VectorIds) {}
 
     /**
-     * @brief List all tables in database
+     * @brief This method is used to query vector in table.
      *
-     * This method is used to list all tables.
+     * @param SearchParam, search parameters.
      *
+     * @return TopKQueryResultList
+     */
+    rpc Search(SearchParam) returns (TopKQueryResultList) {}
+
+    /**
+     * @brief This method is used to query vector in specified files.
      *
-     * @return table names.
+     * @param SearchInFilesParam, search in files paremeters.
+     *
+     * @return TopKQueryResultList
      */
-    rpc ShowTables(Command) returns (TableNameList) {}
+    rpc SearchInFiles(SearchInFilesParam) returns (TopKQueryResultList) {}
 
     /**
-     * @brief Give the server status
+     * @brief This method is used to give the server status.
      *
-     * This method is used to give the server status.
+     * @param Command, command string
      *
-     * @return Server status.
+     * @return StringReply
      */
     rpc Cmd(Command) returns (StringReply) {}
 
      /**
-     * @brief delete table by range
+     * @brief This method is used to delete vector by date range
      *
-     * This method is used to delete vector by range
+     * @param DeleteByDateParam, delete parameters.
      *
-     * @return rpc status.
+     * @return status
      */
-     rpc DeleteByRange(DeleteByRangeParam) returns (Status) {}
+     rpc DeleteByDate(DeleteByDateParam) returns (Status) {}
 
      /**
-      * @brief preload table
+      * @brief This method is used to preload table
       *
-      * This method is used to preload table
+      * @param TableName, target table name.
       *
-      * @return Status.
+      * @return Status
       */
      rpc PreloadTable(TableName) returns (Status) {}
-
-     /**
-      * @brief describe index
-      *
-      * This method is used to describe index
-      *
-      * @return Status.
-      */
-     rpc DescribeIndex(TableName) returns (IndexParam) {}
-
-     /**
-      * @brief drop index
-      *
-      * This method is used to drop index
-      *
-      * @return Status.
-      */
-     rpc DropIndex(TableName) returns (Status) {}
-
 }

core/src/index/CMakeLists.txt

@@ -18,11 +18,11 @@
 #-------------------------------------------------------------------------------
 
 
-cmake_minimum_required(VERSION 3.14)
+cmake_minimum_required(VERSION 3.12)
 message(STATUS "------------------------------KNOWHERE-----------------------------------")
 message(STATUS "Building using CMake version: ${CMAKE_VERSION}")
 
-set(KNOWHERE_VERSION "0.5.0")
+set(KNOWHERE_VERSION "0.6.0")
 string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" KNOWHERE_BASE_VERSION "${KNOWHERE_VERSION}")
 project(knowhere VERSION "${KNOWHERE_BASE_VERSION}" LANGUAGES C CXX)
 set(CMAKE_CXX_STANDARD 14)
@@ -72,17 +72,17 @@ include(ExternalProject)
 include(DefineOptionsCore)
 include(BuildUtilsCore)
 
-set(KNOWHERE_GPU_VERSION false)
-if (MILVUS_CPU_VERSION OR KNOWHERE_CPU_VERSION)
-    message(STATUS "Building Knowhere CPU version")
-    add_compile_definitions("MILVUS_CPU_VERSION")
-else ()
+set(KNOWHERE_CPU_VERSION false)
+if (MILVUS_GPU_VERSION OR KNOWHERE_GPU_VERSION)
     message(STATUS "Building Knowhere GPU version")
     add_compile_definitions("MILVUS_GPU_VERSION")
-    set(KNOWHERE_GPU_VERSION true)
     enable_language(CUDA)
     find_package(CUDA 10 REQUIRED)
     set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES --expt-extended-lambda")
+else ()
+    message(STATUS "Building Knowhere CPU version")
+    set(KNOWHERE_CPU_VERSION true)
+    add_compile_definitions("MILVUS_CPU_VERSION")
 endif ()
 
 include(ThirdPartyPackagesCore)

core/src/index/build.sh

@@ -5,7 +5,7 @@ BUILD_UNITTEST="OFF"
 INSTALL_PREFIX=$(pwd)/cmake_build
 MAKE_CLEAN="OFF"
 PROFILING="OFF"
-BUILD_FAISS_WITH_MKL="OFF"
+FAISS_WITH_MKL="OFF"
 USE_JFROG_CACHE="OFF"
 
 while getopts "p:d:t:uhrcgmj" arg
@@ -31,7 +31,7 @@ do
                 PROFILING="ON"
                 ;;
              m)
-                BUILD_FAISS_WITH_MKL="ON"
+                FAISS_WITH_MKL="ON"
                 ;;
              j)
                 USE_JFROG_CACHE="ON"
@@ -74,7 +74,7 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then
     -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
     -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
     -DMILVUS_ENABLE_PROFILING=${PROFILING} \
-    -DBUILD_FAISS_WITH_MKL=${BUILD_FAISS_WITH_MKL} \
+    -DFAISS_WITH_MKL=${FAISS_WITH_MKL} \
     -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
     ../"
     echo ${CMAKE_CMD}

core/src/index/cmake/DefineOptionsCore.cmake

@@ -41,18 +41,18 @@ macro(define_option_string name description default)
 endmacro()
 
 #----------------------------------------------------------------------
-set_option_category("CPU version")
+set_option_category("GPU version")
 
-if (MILVUS_CPU_VERSION)
-    define_option(KNOWHERE_CPU_VERSION "Build CPU version only" ON)
+if (MILVUS_GPU_VERSION)
+    define_option(KNOWHERE_GPU_VERSION "Build GPU version" ON)
 else ()
-    define_option(KNOWHERE_CPU_VERSION "Build CPU version only" OFF)
+    define_option(KNOWHERE_GPU_VERSION "Build GPU version" OFF)
 endif ()
 
 #----------------------------------------------------------------------
 set_option_category("Thirdparty")
 
-set(KNOWHERE_DEPENDENCY_SOURCE_DEFAULT "AUTO")
+set(KNOWHERE_DEPENDENCY_SOURCE_DEFAULT "BUNDLED")
 
 define_option_string(KNOWHERE_DEPENDENCY_SOURCE
         "Method to use for acquiring KNOWHERE's build dependencies"
@@ -79,18 +79,7 @@ define_option(KNOWHERE_WITH_FAISS "Build with FAISS library" ON)
 
 define_option(KNOWHERE_WITH_FAISS_GPU_VERSION "Build with FAISS GPU version" ON)
 
-define_option(BUILD_FAISS_WITH_MKL "Build FAISS with MKL" OFF)
-
-#----------------------------------------------------------------------
-if (MSVC)
-    set_option_category("MSVC")
-
-    define_option(MSVC_LINK_VERBOSE
-            "Pass verbose linking options when linking libraries and executables"
-            OFF)
-
-    define_option(KNOWHERE_USE_STATIC_CRT "Build KNOWHERE with statically linked CRT" OFF)
-endif ()
+define_option(FAISS_WITH_MKL "Build FAISS with MKL" OFF)
 
 #----------------------------------------------------------------------
 set_option_category("Test and benchmark")

core/src/index/cmake/FindFAISS.cmake

+set(FAISS_STATIC_LIB_NAME ${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX})
+
+# First, find via if specified FAISS_ROOT
+if (FAISS_ROOT)
+    find_library(FAISS_STATIC_LIB
+            NAMES ${FAISS_STATIC_LIB_NAME}
+            PATHS ${FAISS_ROOT}
+            PATH_SUFFIXES "lib"
+            NO_DEFAULT_PATH
+            )
+    find_path(FAISS_INCLUDE_DIR
+            NAMES "faiss/Index.h"
+            PATHS ${FAISS_ROOT}
+            NO_DEFAULT_PATH
+            PATH_SUFFIXES "include"
+            )
+endif ()
+
+find_package_handle_standard_args(FAISS REQUIRED_VARS FAISS_STATIC_LIB FAISS_INCLUDE_DIR)
+
+if (FAISS_FOUND)
+    if (NOT TARGET faiss)
+        add_library(faiss STATIC IMPORTED)
+
+        set_target_properties(
+                faiss
+                PROPERTIES
+                IMPORTED_LOCATION "${FAISS_STATIC_LIB}"
+                INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}"
+        )
+
+        if (FAISS_WITH_MKL)
+            set_target_properties(
+                    faiss
+                    PROPERTIES
+                    INTERFACE_LINK_LIBRARIES "${MKL_LIBS}")
+        else ()
+            set_target_properties(
+                    faiss
+                    PROPERTIES
+                    INTERFACE_LINK_LIBRARIES "openblas;lapack")
+        endif ()
+    endif ()
+endif ()

core/src/index/knowhere/CMakeLists.txt

@@ -49,12 +49,12 @@ set(depend_libs
         SPTAGLibStatic
         faiss
         arrow
-        ${ARROW_PREFIX}/lib/libjemalloc_pic.a
+        ${ARROW_LIB_DIR}/libjemalloc_pic.a
         gomp
         gfortran
         pthread
         )
-if (BUILD_FAISS_WITH_MKL)
+if (FAISS_WITH_MKL)
     set(depend_libs ${depend_libs}
             "-Wl,--start-group \
             ${MKL_LIB_PATH}/libmkl_intel_ilp64.a \

core/src/index/knowhere/knowhere/common/Log.h

@@ -17,7 +17,7 @@
 
 #pragma once
 
-#include "external/easyloggingpp/easylogging++.h"
+#include "easyloggingpp/easylogging++.h"
 
 namespace knowhere {
 

core/src/index/thirdparty/versions.txt

-ARROW_VERSION=apache-arrow-0.14.0
+ARROW_VERSION=apache-arrow-0.15.1
 BOOST_VERSION=1.70.0
 GTEST_VERSION=1.8.1
 LAPACK_VERSION=v3.8.0
 OPENBLAS_VERSION=v0.3.6
-FAISS_VERSION=branch-0.3.0
\ No newline at end of file
+FAISS_VERSION=1.6.0
+MKL_VERSION=2019.5.281

core/src/index/unittest/faiss_benchmark/CMakeLists.txt

@@ -13,9 +13,9 @@ if (KNOWHERE_GPU_VERSION)
 
     set(depend_libs
             faiss hdf5
-            arrow ${ARROW_PREFIX}/lib/libjemalloc_pic.a
+            arrow ${ARROW_LIB_DIR}/libjemalloc_pic.a
             )
-    if (BUILD_FAISS_WITH_MKL)
+    if (FAISS_WITH_MKL)
         set(depend_libs ${depend_libs}
                 "-Wl,--start-group \
                 ${MKL_LIB_PATH}/libmkl_intel_ilp64.a \