Merge branch 'branch-0.3.1' into 'branch-0.3.1'

MS-241: Build Faiss with MKL if using Intel CPU; else build with OpenBlas . MS-242: clean up cmake and change MAKE_BUILD_ARGS to be user defined variable

See merge request megasearch/milvus!230

Former-commit-id: 7fc20dbf175a1bd8fef82bf470c6a499b2e35080

ci/main_jenkinsfile

@@ -35,7 +35,7 @@ pipeline {
                             defaultContainer 'jnlp'
                             containerTemplate {
                                 name 'milvus-build-env'
-                                image 'registry.zilliz.com/milvus/milvus-build-env:v0.10'
+                                image 'registry.zilliz.com/milvus/milvus-build-env:v0.11'
                                 ttyEnabled true
                                 command 'cat'
                             }

ci/main_jenkinsfile_no_ut

@@ -35,7 +35,7 @@ pipeline {
                             defaultContainer 'jnlp'
                             containerTemplate {
                                 name 'milvus-build-env'
-                                image 'registry.zilliz.com/milvus/milvus-build-env:v0.10'
+                                image 'registry.zilliz.com/milvus/milvus-build-env:v0.11'
                                 ttyEnabled true
                                 command 'cat'
                             }

ci/nightly_main_jenkinsfile

@@ -35,7 +35,7 @@ pipeline {
                             defaultContainer 'jnlp'
                             containerTemplate {
                                 name 'milvus-build-env'
-                                image 'registry.zilliz.com/milvus/milvus-build-env:v0.10'
+                                image 'registry.zilliz.com/milvus/milvus-build-env:v0.11'
                                 ttyEnabled true
                                 command 'cat'
                             }

cpp/CHANGELOG.md

@@ -28,6 +28,8 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-206 - Support SQ8 index type
 - MS-208 - Add buildinde interface for C++ SDK
 - MS-212 - Support Inner product metric type
+- MS-241 - Build Faiss with MKL if using Intel CPU; else build with OpenBlas
+- MS-242 - clean up cmake and change MAKE_BUILD_ARGS to be user defined variable
 
 ## New Feature
 - MS-180 - Add new mem manager

cpp/README.md

 ### Compilation
 #### Step 1: install necessery tools
 
-    Install MySQL
-    
     centos7 : 
-        yum install gfortran qt4 flex bison mysql-devel
+        yum install gfortran qt4 flex bison mysql-devel mysql
         
     ubuntu16.04 : 
-        sudo apt-get install gfortran qt4-qmake flex bison libmysqlclient-dev
+        sudo apt-get install gfortran qt4-qmake flex bison libmysqlclient-dev mysql-client
+        
+    cd scripts && sudo ./requirements.sh     
 
 If `libmysqlclient_r.so` does not exist after installing MySQL Development Files, you need to create a symbolic link:
 

cpp/cmake/DefineOptions.cmake

@@ -57,8 +57,6 @@ define_option(MILVUS_VERBOSE_THIRDPARTY_BUILD
 
 define_option(MILVUS_WITH_ARROW "Build with ARROW" OFF)
 
-define_option(MILVUS_BOOST_USE_SHARED "Rely on boost shared libraries where relevant" OFF)
-
 define_option(MILVUS_BOOST_VENDORED "Use vendored Boost instead of existing Boost. \
 Note that this requires linking Boost statically" ON)
 

cpp/scripts/requirements.sh

+#!/usr/bin/env bash
+
+wget -P /tmp https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
+apt-key add /tmp/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
+
+sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list'
+apt -y update && apt-get -y install intel-mkl-gnu-2019.4-243 intel-mkl-core-2019.4-243
+
+#sh -c 'echo export LD_LIBRARY_PATH=/opt/intel/compilers_and_libraries_2019.4.243/linux/mkl/lib/intel64:\$LD_LIBRARY_PATH > /etc/profile.d/mkl.sh'
+#source /etc/profile

cpp/src/CMakeLists.txt

@@ -48,7 +48,6 @@ set(engine_files
         ${db_files}
         ${db_scheduler_files}
         ${wrapper_files}
-#        metrics/Metrics.cpp
         ${metrics_files}
         )
 
@@ -71,8 +70,6 @@ set(third_party_libs
         yaml-cpp
         libgpufaiss.a
         faiss
-        lapack
-        openblas
         prometheus-cpp-push
         prometheus-cpp-pull
         prometheus-cpp-core
@@ -87,9 +84,19 @@ set(third_party_libs
         mysqlpp
         ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so
         )
+
 if (MEGASEARCH_WITH_ARROW STREQUAL "ON")
     set(third_party_libs ${third_party_libs} arrow)
-endif() 
+endif()
+if(${BUILD_FAISS_WITH_MKL} STREQUAL "true")
+    set(third_party_libs ${third_party_libs}
+            ${MKL_LIBS}
+            ${MKL_LIBS})
+else()
+    set(third_party_libs ${third_party_libs}
+            lapack
+            openblas)
+endif()
 
 if (GPU_VERSION STREQUAL "ON")
     link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
@@ -187,6 +194,6 @@ install(FILES
         ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}
         ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3
         ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3.2.4
-        DESTINATION lib) #need to copy libmysqlpp.so
+        DESTINATION lib)
 
 add_subdirectory(sdk)

cpp/thirdparty/versions.txt

@@ -3,6 +3,7 @@ BOOST_VERSION=1.70.0
 BZIP2_VERSION=1.0.6
 EASYLOGGINGPP_VERSION=v9.96.7
 FAISS_VERSION=7b07685
+MKL_VERSION=2019.4.243
 GTEST_VERSION=1.8.1
 JSONCONS_VERSION=0.126.0
 LAPACK_VERSION=v3.8.0

cpp/unittest/CMakeLists.txt

@@ -28,7 +28,6 @@ set(unittest_libs
         easyloggingpp
         pthread
         metrics
-        openblas
         gfortran
         prometheus-cpp-pull
         prometheus-cpp-push

cpp/unittest/db/CMakeLists.txt

@@ -23,10 +23,7 @@ link_directories("/usr/local/cuda/lib64")
 
 include_directories(/usr/include/mysql)
 
-#add_definitions(-DBOOST_ERROR_CODE_HEADER_ONLY)
-
 set(db_test_src
-    #${unittest_srcs}
     ${config_files}
     ${cache_srcs}
     ${db_srcs}
@@ -49,6 +46,11 @@ set(db_libs
         mysqlpp
         )
 
+if(${BUILD_FAISS_WITH_MKL} STREQUAL "true")
+    set(db_libs ${db_libs} ${MKL_LIBS} ${MKL_LIBS})
+endif()
+
 target_link_libraries(db_test ${db_libs} ${unittest_libs})
 
 install(TARGETS db_test DESTINATION bin)
+

cpp/unittest/faiss_wrapper/CMakeLists.txt

@@ -35,6 +35,10 @@ set(wrapper_libs
         zstd
         lz4
         )
+if(${BUILD_FAISS_WITH_MKL} STREQUAL "true")
+    set(wrapper_libs ${wrapper_libs} ${MKL_LIBS} ${MKL_LIBS})
+endif()
+
 target_link_libraries(wrapper_test ${wrapper_libs} ${unittest_libs})
 
 set(topk_test_src

cpp/unittest/metrics/CMakeLists.txt

@@ -10,8 +10,6 @@
 
 include_directories(../../src)
 
-
-
 aux_source_directory(../../src/db db_srcs)
 aux_source_directory(../../src/config config_files)
 aux_source_directory(../../src/cache cache_srcs)
@@ -33,21 +31,10 @@ include_directories(../../third_party/build/include)
 link_directories(../../third_party/build/lib)
 include_directories(/usr/local/cuda/include)
 link_directories("/usr/local/cuda/lib64")
-#include_directories(../db/utils.h)
 include_directories(../../src/metrics)
 
 include_directories(/usr/include/mysql)
 
-#set(metrics_src_files
-#        ../../src/metrics/Metrics.cpp
-#        ../../src/metrics/Metrics.h
-#        ../../src/metrics/PrometheusMetrics.cpp
-#        ../../src/metrics/MetricBase.h
-#        ../../src/server/ServerConfig.cpp
-#        ../../src/utils/CommonUtil.cpp
-#        ../../src/utils/TimeRecorder.cpp
-#        )
-
 set(count_test_src
         ${config_files}
         ${cache_srcs}
@@ -77,5 +64,8 @@ target_link_libraries(metrics_test
         mysqlpp
         ${unittest_libs}
         )
+if(${BUILD_FAISS_WITH_MKL} STREQUAL "true")
+    target_link_libraries(metrics_test ${MKL_LIBS} ${MKL_LIBS})
+endif()
 
 install(TARGETS metrics_test DESTINATION bin)
\ No newline at end of file

cpp/unittest/server/CMakeLists.txt

@@ -48,6 +48,10 @@ set(require_libs
         pthread
         )
 
+if(${BUILD_FAISS_WITH_MKL} STREQUAL "true")
+    set(require_libs ${require_libs} ${MKL_LIBS} ${MKL_LIBS})
+endif()
+
 target_link_libraries(server_test
         ${require_libs}
         ${cuda_library}