Integreation with Velox (#22102)

Signed-off-by: NEnwei Jiao <enwei.jiao@zilliz.com>

.env

 IMAGE_REPO=milvusdb
 IMAGE_ARCH=amd64
 OS_NAME=ubuntu20.04
-DATE_VERSION=20230209-e1ec248
-LATEST_DATE_VERSION=20230209-e1ec248
+DATE_VERSION=20230215-5351fee
+LATEST_DATE_VERSION=20230215-5351fee
 MINIO_ADDRESS=minio:9000
 PULSAR_ADDRESS=pulsar://pulsar:6650
 ETCD_ENDPOINTS=etcd:2379

.github/workflows/code-checker.yaml

@@ -27,7 +27,7 @@ jobs:
   ubuntu:
     name: ${{ matrix.name }}
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 60
+    timeout-minutes: 120
     strategy:
       fail-fast: false
       matrix:
@@ -46,7 +46,8 @@ jobs:
           echo "corehash=${CORE_HASH}" >> $GITHUB_ENV
           echo "Set CCache hash to ${CORE_HASH}"
       - name: Cache CCache Volumes
-        uses: actions/cache@v3
+        # uses: actions/cache@v3
+        uses: pat-s/always-upload-cache@v3
         with:
           path: .docker/amd64-ubuntu20.04-ccache
           key: ubuntu20.04-ccache-${{ env.corehash }}
@@ -58,7 +59,8 @@ jobs:
           key: ubuntu20.04-go-mod-${{ hashFiles('**/go.sum') }}
           restore-keys: ubuntu20.04-go-mod-
       - name: Cache Conan Packages
-        uses: actions/cache@v3
+        # uses: actions/cache@v3
+        uses: pat-s/always-upload-cache@v3
         with:
           path: .docker/amd64-ubuntu20.04-conan
           key: ubuntu20.04-conan-${{ hashFiles('internal/core/conanfile.*') }}
@@ -72,7 +74,7 @@ jobs:
     name: Code Checker CentOS 7
     # Run in centos docker
     runs-on: ubuntu-20.04
-    timeout-minutes: 60
+    timeout-minutes: 120
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -83,7 +85,8 @@ jobs:
           echo "corehash=${CORE_HASH}" >> $GITHUB_ENV
           echo "Set CCache hash to ${CORE_HASH}"
       - name: Cache CCache Volumes
-        uses: actions/cache@v3
+        # uses: actions/cache@v3
+        uses: pat-s/always-upload-cache@v3
         with:
           path: .docker/amd64-centos7-ccache
           key: centos7-ccache-${{ env.corehash }}
@@ -101,11 +104,12 @@ jobs:
           key: centos7-go-mod-${{ hashFiles('**/go.sum') }}
           restore-keys: centos7-go-mod-
       - name: Cache Conan Packages
-        uses: actions/cache@v3
+        # uses: actions/cache@v3
+        uses: pat-s/always-upload-cache@v3
         with:
           path: .docker/amd64-centos7-conan
-          key: centos7-conan-${{ hashFiles('internal/core/conanfile.txt') }}
-          restore-keys: centos7-conan-
+          key: centos7-conan-${{ hashFiles('internal/core/conanfile.*') }}
+          # restore-keys: centos7-conan-
       - name: Code Check
         env:
           OS_NAME: 'centos7'

.github/workflows/mac.yaml

@@ -26,7 +26,7 @@ jobs:
   mac:
     name: Code Checker MacOS 12
     runs-on: macos-12
-    timeout-minutes: 120
+    timeout-minutes: 180
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -37,7 +37,8 @@ jobs:
           echo "corehash=${CORE_HASH}" >> $GITHUB_ENV
           echo "Set CCache hash to ${CORE_HASH}"
       - name: Mac Cache Third Party
-        uses: actions/cache@v3
+        # uses: actions/cache@v3
+        uses: pat-s/always-upload-cache@v3
         with:
           path: |
             /tmp/thirdparty

.github/workflows/main.yaml

 name: Build and test
-# TODO: do not trigger action for some document file update
 
 # This workflow is triggered on pushes or pull request to the repository.
 on:
@@ -36,7 +35,7 @@ jobs:
   UT:
     name: Build and test AMD64 Ubuntu ${{ matrix.ubuntu }}
     runs-on: ubuntu-${{ matrix.ubuntu }}
-    timeout-minutes: 120
+    timeout-minutes: 180
     strategy:
       fail-fast: false
       matrix:
@@ -60,12 +59,12 @@ jobs:
             **/*.h
             **/*.hpp
             **/*.CMakeLists.txt
-            **/conanfile.*          
+            **/conanfile.*
       - name: 'Setup Use USE_ASAN'
         if: steps.changed-files-cpp.outputs.any_changed == 'true'
         run: |
-            echo "useasan=true" >> $GITHUB_ENV
-            echo "Setup USE_ASAN to true since cpp file(s) changed"
+          echo "useasan=true" >> $GITHUB_ENV
+          echo "Setup USE_ASAN to true since cpp file(s) changed"
       - name: 'Generate CCache Hash'
         env:
           CORE_HASH: ${{ hashFiles( 'internal/core/**/*.cpp', 'internal/core/**/*.cc', 'internal/core/**/*.c', 'internal/core/**/*.h', 'internal/core/**/*.hpp', 'internal/core/**/CMakeLists.txt', 'internal/core/conanfile.py') }}

internal/core/CMakeLists.txt

@@ -19,8 +19,6 @@ cmake_minimum_required( VERSION 3.18 )
 if ( APPLE )
     set( CMAKE_CROSSCOMPILING TRUE )
     set( RUN_HAVE_GNU_POSIX_REGEX 0 )
-    set( CMAKE_C_COMPILER "/usr/local/opt/llvm/bin/clang" )
-    set( CMAKE_CXX_COMPILER "/usr/local/opt/llvm/bin/clang++" )
 endif ()
 
 add_definitions(-DELPP_THREAD_SAFE)
@@ -31,7 +29,6 @@ project(core)
 include(CheckCXXCompilerFlag)
 if ( APPLE )
     message(STATUS "==============Darwin Environment==============")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I/usr/local/opt/llvm/include -I/usr/local/include -I/usr/local/opt/libomp/include -L/usr/local/opt/libomp/lib")
 elseif (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
     message(STATUS "==============Linux Environment===============")
     set(LINUX TRUE)
@@ -90,7 +87,6 @@ set( CONAN_DISABLE_CHECK_COMPILER ON )
 conan_basic_setup( KEEP_RPATHS )
 
 set( CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} ${CONAN_PROTOBUF_ROOT})
-message( STATUS "cmake prefix path ${CMAKE_PREFIX_PATH}")
 
 include( CTest )
 include( BuildUtils )

internal/core/conanfile.py

@@ -9,6 +9,9 @@ class MilvusConan(ConanFile):
         "boost/1.81.0",
         "onetbb/2021.7.0",
         "zstd/1.5.2",
+        "lz4/1.9.4",
+        "snappy/1.1.9",
+        "lzo/2.10",
         "arrow/8.0.1",
         "openssl/1.1.1q",
         "aws-sdk-cpp/1.9.234",
@@ -20,8 +23,20 @@ class MilvusConan(ConanFile):
         "marisa/0.2.6",
         "zlib/1.2.13",
         "libcurl/7.86.0",
+        "glog/0.6.0",
+        "fmt/8.0.1",
+        "gflags/2.2.2",
+        "double-conversion/3.2.1",
+        "libevent/2.1.12",
+        "libdwarf/20191104",
+        "libiberty/9.1.0",
+        "libsodium/cci.20220430",
+        "bison/3.5.3",
+        "flex/2.6.4",
+        "xsimd/9.0.1",
+        "folly/2022.10.31.01@milvus/dev",
+        "velox/2023.02.07.01@milvus/dev",
     )
-
     generators = ("cmake", "cmake_find_package")
     default_options = {
         "rocksdb:shared": True,
@@ -33,14 +48,16 @@ class MilvusConan(ConanFile):
         "aws-sdk-cpp:text-to-speech": False,
         "aws-sdk-cpp:transfer": False,
         "gtest:build_gmock": False,
+        "folly:use_sse4_2": True,
     }
-    should_build = False
 
     def configure(self):
         # Macos M1 cannot use jemalloc
         if self.settings.os == "Macos":
             if self.settings.arch not in ("x86_64", "x86"):
-                self.options["arrow"].with_jemalloc = False
+                del self.options["folly"].use_sse4_2
+
+            self.options["arrow"].with_jemalloc = False
             self.options["boost"].without_fiber = True
             self.options["boost"].without_json = True
             self.options["boost"].without_wave = True

internal/core/src/common/CMakeLists.txt

@@ -28,8 +28,6 @@ target_link_libraries(milvus_common
         milvus_log
         yaml-cpp
         boost_bitset_ext
-        # arrow
-        # parquet
         ${CONAN_LIBS}
         )
 

internal/core/src/segcore/CMakeLists.txt

@@ -53,3 +53,7 @@ target_link_libraries(milvus_segcore
         )
 
 install(TARGETS milvus_segcore DESTINATION "${CMAKE_INSTALL_LIBDIR}")
+
+add_executable(velox_demo VeloxDemo.cpp)
+target_link_libraries(velox_demo ${CONAN_LIBS} velox_bundled)
+install(TARGETS velox_demo DESTINATION "${CMAKE_INSTALL_BINDIR}")

internal/core/src/segcore/VeloxDemo.cpp

+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <folly/init/Init.h>
+#include <algorithm>
+
+#include "velox/common/file/FileSystems.h"
+#include "velox/common/memory/Memory.h"
+#include "velox/dwio/dwrf/reader/DwrfReader.h"
+#include "velox/exec/tests/utils/TempDirectoryPath.h"
+#include "velox/vector/BaseVector.h"
+
+using namespace facebook::velox;
+using namespace facebook::velox::dwio::common;
+using namespace facebook::velox::dwrf;
+
+// A temporary program that reads from ORC file and prints its content
+// Used to compare the ORC data read by DWRFReader against apache-orc repo.
+// Usage: velox_example_scan_orc {orc_file_path}
+int
+main(int argc, char** argv) {
+    folly::init(&argc, &argv);
+
+    if (argc < 2) {
+        return 1;
+    }
+
+    // To be able to read local files, we need to register the local file
+    // filesystem. We also need to register the dwrf reader factory:
+    filesystems::registerLocalFileSystem();
+    dwrf::registerDwrfReaderFactory();
+    auto pool = facebook::velox::memory::getDefaultMemoryPool();
+
+    std::string filePath{argv[1]};
+    ReaderOptions readerOpts{pool.get()};
+    // To make DwrfReader reads ORC file, setFileFormat to FileFormat::ORC
+    readerOpts.setFileFormat(FileFormat::ORC);
+    auto reader = DwrfReader::create(
+        std::make_unique<BufferedInput>(std::make_shared<LocalReadFile>(filePath), readerOpts.getMemoryPool()),
+        readerOpts);
+
+    VectorPtr batch;
+    RowReaderOptions rowReaderOptions;
+    auto rowReader = reader->createRowReader(rowReaderOptions);
+    while (rowReader->next(500, batch)) {
+        auto rowVector = batch->as<RowVector>();
+        for (vector_size_t i = 0; i < rowVector->size(); ++i) {
+            std::cout << rowVector->toString(i) << std::endl;
+        }
+    }
+
+    return 0;
+}

internal/core/thirdparty/CMakeLists.txt

@@ -12,37 +12,37 @@
 #-------------------------------------------------------------------------------
 # Using default c and cxx compiler in our build tree
 # Thirdpart cxx and c flags
-add_compile_options( -O3 -fPIC -Wno-error -fopenmp )
+add_compile_options(-O3 -fPIC -Wno-error -fopenmp)
 
-if ( NOT KNOWHERE_VERBOSE_THIRDPARTY_BUILD )
-    set( EP_LOG_OPTIONS LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1 LOG_DOWNLOAD 1 )
+if (NOT KNOWHERE_VERBOSE_THIRDPARTY_BUILD)
+    set(EP_LOG_OPTIONS LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1 LOG_DOWNLOAD 1)
 else ()
-    set( EP_LOG_OPTIONS )
+    set(EP_LOG_OPTIONS)
 endif ()
 
-set( MAKE_BUILD_ARGS "-j6" )
+set(MAKE_BUILD_ARGS "-j6")
 
-include( FetchContent )
-set( FETCHCONTENT_BASE_DIR  ${MILVUS_BINARY_DIR}/3rdparty_download )
-set( FETCHCONTENT_QUIET     OFF )
+include(FetchContent)
+set(FETCHCONTENT_BASE_DIR  ${MILVUS_BINARY_DIR}/3rdparty_download)
+set(FETCHCONTENT_QUIET     OFF)
 
-if( CUSTOM_THIRDPARTY_DOWNLOAD_PATH )
-    set( THIRDPARTY_DOWNLOAD_PATH ${CUSTOM_THIRDPARTY_DOWNLOAD_PATH} )
+if(CUSTOM_THIRDPARTY_DOWNLOAD_PATH)
+    set(THIRDPARTY_DOWNLOAD_PATH ${CUSTOM_THIRDPARTY_DOWNLOAD_PATH})
 else()
-    set( THIRDPARTY_DOWNLOAD_PATH ${CMAKE_BINARY_DIR}/3rdparty_download/download )
+    set(THIRDPARTY_DOWNLOAD_PATH ${CMAKE_BINARY_DIR}/3rdparty_download/download)
 endif()
-message( STATUS "Thirdparty downloaded file path: ${THIRDPARTY_DOWNLOAD_PATH}" )
+message(STATUS "Thirdparty downloaded file path: ${THIRDPARTY_DOWNLOAD_PATH}")
 # ----------------------------------------------------------------------
 # Find pthreads
 
-set( THREADS_PREFER_PTHREAD_FLAG ON )
-find_package( Threads REQUIRED )
+set(THREADS_PREFER_PTHREAD_FLAG ON)
+find_package(Threads REQUIRED)
 
-add_subdirectory( knowhere )
+add_subdirectory(knowhere)
 
-add_subdirectory( boost_ext )
-add_subdirectory( rocksdb )
+add_subdirectory(boost_ext)
+add_subdirectory(rocksdb)
 
-if ( LINUX )
-    add_subdirectory( jemalloc )
-endif()
\ No newline at end of file
+if (LINUX)
+    add_subdirectory(jemalloc)
+endif()

scripts/core_build.sh

@@ -188,6 +188,8 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then
   exit 0
 fi
 
+export CONAN_REVISIONS_ENABLED=1
+conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local
 unameOut="$(uname -s)"
 case "${unameOut}" in
   Darwin*)
@@ -195,15 +197,18 @@ case "${unameOut}" in
     export CLANG_TOOLS_PATH="${llvm_prefix}/bin"
     export CC="${llvm_prefix}/bin/clang"
     export CXX="${llvm_prefix}/bin/clang++"
-    export LDFLAGS="-L${llvm_prefix}/lib -L/usr/local/opt/libomp/lib"
-    export CXXFLAGS="-I${llvm_prefix}/include -I/usr/local/include -I/usr/local/opt/libomp/include"
-    conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler=clang -s compiler.libcxx=libc++ || { echo 'conan install failed'; exit 1; }
+    export CFLAGS=-Wno-deprecated-declarations
+    export CXXFLAGS=-Wno-deprecated-declarations
+    conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler=clang -s compiler.version=15 -s compiler.libcxx=libc++ || { echo 'conan install failed'; exit 1; }
     ;;
   Linux*)
+    echo "Running on ${OS_NAME}"
+    export CPU_TARGET=avx
+    GCC_VERSION=`gcc -dumpversion`
     if [[ `gcc -v 2>&1 | sed -n 's/.*\(--with-default-libstdcxx-abi\)=\(\w*\).*/\2/p'` == "gcc4" ]]; then
-      conan install ${CPP_SRC_DIR} --install-folder conan --build=missing || { echo 'conan install failed'; exit 1; }
+      conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler.version=${GCC_VERSION} || { echo 'conan install failed'; exit 1; }
     else 
-      conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler.libcxx=libstdc++11 || { echo 'conan install failed'; exit 1; }
+      conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler.version=${GCC_VERSION} -s compiler.libcxx=libstdc++11 || { echo 'conan install failed'; exit 1; }
     fi 
     ;;
   *)   
@@ -211,6 +216,7 @@ case "${unameOut}" in
     ;;
 esac
 
+arch=$(uname -m)
 CMAKE_CMD="cmake \
 ${CMAKE_EXTRA_ARGS} \
 -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
@@ -218,6 +224,7 @@ ${CMAKE_EXTRA_ARGS} \
 -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
 -DOpenBLAS_SOURCE=AUTO \
 -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
+-DCMAKE_LIBRARY_ARCHITECTURE=${arch} \
 -DBUILD_COVERAGE=${BUILD_COVERAGE} \
 -DMILVUS_DB_PATH=${DB_PATH} \
 -DENABLE_CPU_PROFILING=${PROFILING} \