未验证 提交 5950dedd 编写于 作者: C Cai Yudong 提交者: GitHub

#1678 remove CUSTOMIZATION macro (#1707)

* #1678 remove CUSTOMIZATION macro
Signed-off-by: Nyudong.cai <yudong.cai@zilliz.com>

* disable faiss_benchmark
Signed-off-by: Nyudong.cai <yudong.cai@zilliz.com>
上级 588ef95d
......@@ -26,6 +26,7 @@ Please mark all change in change log and use the issue from GitHub
- \#1619 Improve compact performance
- \#1649 Fix Milvus crash on old CPU
- \#1653 IndexFlat (SSE) and IndexBinaryFlat performance improvement for small NQ
- \#1678 Remove CUSTOMIZATION macro
## Task
......
......@@ -153,10 +153,6 @@ if (MILVUS_USE_CCACHE)
endif (CCACHE_FOUND)
endif ()
if (CUSTOMIZATION)
add_compile_definitions(CUSTOMIZATION)
endif ()
if (MILVUS_GPU_VERSION)
message(STATUS "Building Milvus GPU version")
add_compile_definitions("MILVUS_GPU_VERSION")
......
......@@ -45,8 +45,6 @@ set_option_category("Milvus Build Option")
define_option(MILVUS_GPU_VERSION "Build GPU version" OFF)
define_option(CUSTOMIZATION "Build with customized FAISS library" ON)
#----------------------------------------------------------------------
set_option_category("Thirdparty")
......
......@@ -186,13 +186,11 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
index = vec_index_factory.CreateVecIndex(knowhere::IndexEnum::INDEX_FAISS_IVFSQ8, mode);
break;
}
#ifdef CUSTOMIZATION
#ifdef MILVUS_GPU_VERSION
case EngineType::FAISS_IVFSQ8H: {
index = vec_index_factory.CreateVecIndex(knowhere::IndexEnum::INDEX_FAISS_IVFSQ8H, mode);
break;
}
#endif
#endif
case EngineType::FAISS_BIN_IDMAP: {
index = vec_index_factory.CreateVecIndex(knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP, mode);
......
......@@ -66,11 +66,6 @@ include(ExternalProject)
include(DefineOptionsCore)
include(BuildUtilsCore)
if (CUSTOMIZATION)
add_compile_definitions(CUSTOMIZATION)
endif ()
set(KNOWHERE_CPU_VERSION false)
if (MILVUS_GPU_VERSION OR KNOWHERE_GPU_VERSION)
message(STATUS "Building Knowhere GPU version")
add_compile_definitions("MILVUS_GPU_VERSION")
......@@ -79,8 +74,6 @@ if (MILVUS_GPU_VERSION OR KNOWHERE_GPU_VERSION)
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES --expt-extended-lambda")
else ()
message(STATUS "Building Knowhere CPU version")
set(KNOWHERE_CPU_VERSION true)
add_compile_definitions("MILVUS_CPU_VERSION")
endif ()
include(ThirdPartyPackagesCore)
......
......@@ -49,8 +49,6 @@ else ()
define_option(KNOWHERE_GPU_VERSION "Build GPU version" OFF)
endif ()
define_option(CUSTOMIZATION "Build with customized FAISS library" ON)
#----------------------------------------------------------------------
set_option_category("Thirdparty")
......
......@@ -301,7 +301,7 @@ IVF::QueryImpl(int64_t n, const float* data, int64_t k, float* distances, int64_
void
IVF::SealImpl() {
#ifdef CUSTOMIZATION
#ifdef MILVUS_GPU_VERSION
faiss::Index* index = index_.get();
auto idx = dynamic_cast<faiss::IndexIVF*>(index);
if (idx != nullptr) {
......
......@@ -26,7 +26,7 @@
namespace milvus {
namespace knowhere {
#ifdef CUSTOMIZATION
#ifdef MILVUS_GPU_VERSION
void
IVFSQHybrid::Train(const DatasetPtr& dataset_ptr, const Config& config) {
......
......@@ -23,7 +23,8 @@
namespace milvus {
namespace knowhere {
#ifdef CUSTOMIZATION
#ifdef MILVUS_GPU_VERSION
struct FaissIVFQuantizer : public Quantizer {
faiss::gpu::GpuIndexFlat* quantizer = nullptr;
int64_t gpu_id;
......
......@@ -9,6 +9,7 @@
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#ifdef MILVUS_GPU_VERSION
#include "knowhere/index/vector_index/helpers/Cloner.h"
#include "knowhere/common/Exception.h"
#include "knowhere/index/vector_index/IndexIDMAP.h"
......@@ -39,13 +40,11 @@ VecIndexPtr
CopyCpuToGpu(const VecIndexPtr& index, const int64_t device_id, const Config& config) {
VecIndexPtr result;
auto uids = index->GetUids();
#ifdef CUSTOMIZATION
if (auto device_index = std::dynamic_pointer_cast<IVFSQHybrid>(index)) {
result = device_index->CopyCpuToGpu(device_id, config);
result->SetUids(uids);
return result;
}
#endif
if (auto device_index = std::dynamic_pointer_cast<GPUIndex>(index)) {
result = device_index->CopyGpuToGpu(device_id, config);
......@@ -72,3 +71,4 @@ CopyCpuToGpu(const VecIndexPtr& index, const int64_t device_id, const Config& co
} // namespace cloner
} // namespace knowhere
} // namespace milvus
#endif
......@@ -9,10 +9,7 @@
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#define USE_FAISS_V_0_3_0
#include <gtest/gtest.h>
#include <hdf5.h>
#include <sys/time.h>
#include <sys/types.h>
......@@ -25,32 +22,15 @@
#include <faiss/AutoTune.h>
#include <faiss/Index.h>
#include <faiss/IndexIVF.h>
#include <faiss/gpu/GpuCloner.h>
#include <faiss/gpu/GpuIndexFlat.h>
#include <faiss/gpu/GpuIndexIVF.h>
#include <faiss/gpu/GpuIndexIVFSQHybrid.h>
#include <faiss/gpu/StandardGpuResources.h>
#include <faiss/index_io.h>
#ifdef USE_FAISS_V_0_3_0 // faiss_0.3.0
#include <faiss/gpu/GpuCloner.h>
#include <faiss/index_factory.h>
#include <faiss/index_io.h>
#include <faiss/utils/distances.h>
#else // faiss_0.2.1
#include <faiss/gpu/GpuAutoTune.h>
#include <faiss/utils.h>
#include <sys/stat.h>
#include <cstdlib>
#include <cstring>
#endif
#ifdef CUSTOMIZATION
#include <faiss/gpu/GpuIndexIVFSQHybrid.h>
#else
#include <faiss/gpu/GpuIndexIVF.h>
#endif
/*****************************************************
* To run this test, please download the HDF5 from
* https://support.hdfgroup.org/ftp/HDF5/releases/
......@@ -291,12 +271,10 @@ load_base_data(faiss::Index*& index, const std::string& ann_test_name, const std
cpu_index = faiss::gpu::index_gpu_to_cpu(gpu_index);
delete gpu_index;
#ifdef CUSTOMIZATION
faiss::IndexIVF* cpu_ivf_index = dynamic_cast<faiss::IndexIVF*>(cpu_index);
if (cpu_ivf_index != nullptr) {
cpu_ivf_index->to_readonly();
}
#endif
printf("[%.3f s] Writing index file: %s\n", elapsed() - t0, index_file_name.c_str());
faiss::write_index(cpu_index, index_file_name.c_str());
......@@ -372,15 +350,12 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key
faiss::Index *gpu_index, *index;
if (query_mode != MODE_CPU) {
faiss::gpu::GpuClonerOptions option;
#ifdef CUSTOMIZATION
option.allInGpu = true;
faiss::IndexComposition index_composition;
index_composition.index = cpu_index;
index_composition.quantizer = nullptr;
#endif
switch (query_mode) {
#ifdef CUSTOMIZATION
case MODE_MIX: {
index_composition.mode = 1; // 0: all data, 1: copy quantizer, 2: copy data
......@@ -403,9 +378,8 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key
index = cpu_index;
break;
}
#endif
case MODE_GPU:
#ifdef CUSTOMIZATION
#if 1
index_composition.mode = 0; // 0: all data, 1: copy quantizer, 2: copy data
// warm up the transmission
......@@ -571,14 +545,12 @@ TEST(FAISSTEST, BENCHMARK) {
test_ann_hdf5("sift-128-euclidean", "IVF16384", "SQ8", MODE_CPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
test_ann_hdf5("sift-128-euclidean", "IVF16384", "SQ8", MODE_GPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
#ifdef CUSTOMIZATION
test_ann_hdf5("sift-128-euclidean", "IVF16384", "SQ8Hybrid", MODE_CPU, SIFT_INSERT_LOOPS, param_nprobes,
SEARCH_LOOPS);
test_ann_hdf5("sift-128-euclidean", "IVF16384", "SQ8Hybrid", MODE_MIX, SIFT_INSERT_LOOPS, param_nprobes,
SEARCH_LOOPS);
test_ann_hdf5("sift-128-euclidean", "IVF16384", "SQ8Hybrid", MODE_GPU, SIFT_INSERT_LOOPS, param_nprobes,
SEARCH_LOOPS);
#endif
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
const int32_t GLOVE_INSERT_LOOPS = 1;
......@@ -589,12 +561,10 @@ TEST(FAISSTEST, BENCHMARK) {
test_ann_hdf5("glove-200-angular", "IVF16384", "SQ8", MODE_CPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
test_ann_hdf5("glove-200-angular", "IVF16384", "SQ8", MODE_GPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
#ifdef CUSTOMIZATION
test_ann_hdf5("glove-200-angular", "IVF16384", "SQ8Hybrid", MODE_CPU, GLOVE_INSERT_LOOPS, param_nprobes,
SEARCH_LOOPS);
test_ann_hdf5("glove-200-angular", "IVF16384", "SQ8Hybrid", MODE_MIX, GLOVE_INSERT_LOOPS, param_nprobes,
SEARCH_LOOPS);
test_ann_hdf5("glove-200-angular", "IVF16384", "SQ8Hybrid", MODE_GPU, GLOVE_INSERT_LOOPS, param_nprobes,
SEARCH_LOOPS);
#endif
}
......@@ -29,21 +29,16 @@
#include <faiss/AutoTune.h>
#include <faiss/Index.h>
#include <faiss/IndexIVF.h>
#include <faiss/gpu/GpuCloner.h>
#include <faiss/gpu/GpuIndexFlat.h>
#include <faiss/gpu/GpuIndexIVF.h>
#include <faiss/gpu/GpuIndexIVFSQHybrid.h>
#include <faiss/gpu/StandardGpuResources.h>
#include <faiss/index_io.h>
#include <faiss/gpu/GpuCloner.h>
#include <faiss/index_factory.h>
#include <faiss/index_io.h>
#include <faiss/utils/ConcurrentBitset.h>
#include <faiss/utils/distances.h>
#ifdef CUSTOMIZATION
#include <faiss/gpu/GpuIndexIVFSQHybrid.h>
#else
#include <faiss/gpu/GpuIndexIVF.h>
#endif
/*****************************************************
* To run this test, please download the HDF5 from
* https://support.hdfgroup.org/ftp/HDF5/releases/
......@@ -284,12 +279,10 @@ load_base_data(faiss::Index*& index, const std::string& ann_test_name, const std
cpu_index = faiss::gpu::index_gpu_to_cpu(gpu_index);
delete gpu_index;
#ifdef CUSTOMIZATION
faiss::IndexIVF* cpu_ivf_index = dynamic_cast<faiss::IndexIVF*>(cpu_index);
if (cpu_ivf_index != nullptr) {
cpu_ivf_index->to_readonly();
}
#endif
printf("[%.3f s] Writing index file: %s\n", elapsed() - t0, index_file_name.c_str());
faiss::write_index(cpu_index, index_file_name.c_str());
......@@ -381,15 +374,12 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key
faiss::Index *gpu_index, *index;
if (query_mode != MODE_CPU) {
faiss::gpu::GpuClonerOptions option;
#ifdef CUSTOMIZATION
option.allInGpu = true;
faiss::IndexComposition index_composition;
index_composition.index = cpu_index;
index_composition.quantizer = nullptr;
#endif
switch (query_mode) {
#ifdef CUSTOMIZATION
case MODE_MIX: {
index_composition.mode = 1; // 0: all data, 1: copy quantizer, 2: copy data
......@@ -412,9 +402,8 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key
index = cpu_index;
break;
}
#endif
case MODE_GPU:
#ifdef CUSTOMIZATION
#if 1
index_composition.mode = 0; // 0: all data, 1: copy quantizer, 2: copy data
// warm up the transmission
......
......@@ -42,7 +42,7 @@ class SingleIndexTest : public DataGen, public TestGpuIndexBase {
milvus::knowhere::IVFPtr index_ = nullptr;
};
#ifdef CUSTOMIZATION
#ifdef MILVUS_GPU_VERSION
TEST_F(SingleIndexTest, IVFSQHybrid) {
assert(!xb.empty());
......
......@@ -84,9 +84,7 @@ INSTANTIATE_TEST_CASE_P(
std::make_tuple(milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, milvus::knowhere::IndexMode::MODE_GPU),
std::make_tuple(milvus::knowhere::IndexEnum::INDEX_FAISS_IVFPQ, milvus::knowhere::IndexMode::MODE_GPU),
std::make_tuple(milvus::knowhere::IndexEnum::INDEX_FAISS_IVFSQ8, milvus::knowhere::IndexMode::MODE_GPU),
#ifdef CUSTOMIZATION
std::make_tuple(milvus::knowhere::IndexEnum::INDEX_FAISS_IVFSQ8H, milvus::knowhere::IndexMode::MODE_GPU),
#endif
#endif
std::make_tuple(milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, milvus::knowhere::IndexMode::MODE_CPU),
std::make_tuple(milvus::knowhere::IndexEnum::INDEX_FAISS_IVFPQ, milvus::knowhere::IndexMode::MODE_CPU),
......@@ -255,11 +253,9 @@ TEST_P(IVFTest, clone_test) {
}
}
}
#endif
#ifdef MILVUS_GPU_VERSION
#ifdef CUSTOMIZATION
TEST_P(IVFTest, gpu_seal_test) {
if (index_mode_ != milvus::knowhere::IndexMode::MODE_GPU) {
return;
......@@ -300,8 +296,6 @@ TEST_P(IVFTest, gpu_seal_test) {
ASSERT_ANY_THROW(milvus::knowhere::cloner::CopyCpuToGpu(cpu_idx, -1, milvus::knowhere::Config()));
}
#endif
TEST_P(IVFTest, invalid_gpu_source) {
if (index_mode_ != milvus::knowhere::IndexMode::MODE_GPU) {
return;
......@@ -337,7 +331,6 @@ TEST_P(IVFTest, invalid_gpu_source) {
ASSERT_ANY_THROW(index_->Train(base_dataset, invalid_conf));
}
#ifdef CUSTOMIZATION
TEST_P(IVFTest, IVFSQHybrid_test) {
if (index_type_ != milvus::knowhere::IndexEnum::INDEX_FAISS_IVFSQ8H) {
return;
......@@ -360,5 +353,3 @@ TEST_P(IVFTest, IVFSQHybrid_test) {
ASSERT_ANY_THROW(index->SetQuantizer(nullptr));
}
#endif
#endif
......@@ -62,14 +62,12 @@ INSTANTIATE_TEST_CASE_P(
std::make_tuple(milvus::knowhere::IndexType::INDEX_FAISS_IVFFLAT, milvus::knowhere::IndexMode::MODE_GPU),
std::make_tuple(milvus::knowhere::IndexType::INDEX_FAISS_IVFPQ, milvus::knowhere::IndexMode::MODE_GPU),
std::make_tuple(milvus::knowhere::IndexType::INDEX_FAISS_IVFSQ8, milvus::knowhere::IndexMode::MODE_GPU),
#ifdef CUSTOMIZATION
std::make_tuple(milvus::knowhere::IndexType::INDEX_FAISS_IVFSQ8H, milvus::knowhere::IndexMode::MODE_GPU),
#endif
#endif
std::make_tuple(milvus::knowhere::IndexType::INDEX_FAISS_IVFFLAT, milvus::knowhere::IndexMode::MODE_CPU),
std::make_tuple(milvus::knowhere::IndexType::INDEX_FAISS_IVFPQ, milvus::knowhere::IndexMode::MODE_CPU),
std::make_tuple(milvus::knowhere::IndexType::INDEX_FAISS_IVFSQ8, milvus::knowhere::IndexMode::MODE_CPU)
std::make_tuple(milvus::knowhere::IndexType::INDEX_NSG, milvus::knowhere::IndexMode::MODE_CPU),
std::make_tuple(milvus::knowhere::IndexType::INDEX_FAISS_IVFSQ8, milvus::knowhere::IndexMode::MODE_CPU),
std::make_tuple(milvus::knowhere::IndexType::INDEX_NSG, milvus::knowhere::IndexMode::MODE_CPU),
std::make_tuple(milvus::knowhere::IndexType::INDEX_HNSW, milvus::knowhere::IndexMode::MODE_CPU),
std::make_tuple(milvus::knowhere::IndexType::INDEX_SPTAG_KDT_RNT, milvus::knowhere::IndexMode::MODE_CPU),
std::make_tuple(milvus::knowhere::IndexType::INDEX_SPTAG_BKT_RNT, milvus::knowhere::IndexMode::MODE_CPU)));
......
......@@ -123,9 +123,7 @@ class OptimizerInst {
pass_list.push_back(std::make_shared<FaissFlatPass>());
pass_list.push_back(std::make_shared<FaissIVFFlatPass>());
pass_list.push_back(std::make_shared<FaissIVFSQ8Pass>());
#ifdef CUSTOMIZATION
pass_list.push_back(std::make_shared<FaissIVFSQ8HPass>());
#endif
pass_list.push_back(std::make_shared<FaissIVFPQPass>());
}
#endif
......
......@@ -8,6 +8,7 @@
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#ifdef MILVUS_GPU_VERSION
#include "scheduler/optimizer/FaissIVFSQ8HPass.h"
#include "cache/GpuCacheMgr.h"
......@@ -23,7 +24,6 @@ namespace scheduler {
void
FaissIVFSQ8HPass::Init() {
#ifdef CUSTOMIZATION
server::Config& config = server::Config::GetInstance();
Status s = config.GetEngineConfigGpuSearchThreshold(threshold_);
if (!s.ok()) {
......@@ -38,12 +38,10 @@ FaissIVFSQ8HPass::Init() {
AddGpuEnableListener();
AddGpuSearchThresholdListener();
AddGpuSearchResourcesListener();
#endif
}
bool
FaissIVFSQ8HPass::Run(const TaskPtr& task) {
#ifdef CUSTOMIZATION
if (task->Type() != TaskType::SearchTask) {
return false;
}
......@@ -72,7 +70,6 @@ FaissIVFSQ8HPass::Run(const TaskPtr& task) {
auto label = std::make_shared<SpecResLabel>(res_ptr);
task->label() = label;
return true;
#endif
}
} // namespace scheduler
......
......@@ -11,6 +11,7 @@
#include "utils/ValidationUtil.h"
#include "Log.h"
#include "db/Types.h"
#include "db/Utils.h"
#include "db/engine/ExecutionEngine.h"
#include "knowhere/index/vector_index/ConfAdapter.h"
......@@ -20,13 +21,10 @@
#include <arpa/inet.h>
#ifdef MILVUS_GPU_VERSION
#include <cuda_runtime.h>
#endif
#include <fiu-local.h>
#include <src/db/Types.h>
#include <algorithm>
#include <cmath>
#include <regex>
......@@ -169,7 +167,7 @@ ValidationUtil::ValidateTableIndexType(int32_t index_type) {
return Status(SERVER_INVALID_INDEX_TYPE, msg);
}
#ifndef CUSTOMIZATION
#ifndef MILVUS_GPU_VERSION
// special case, hybird index only available in customize faiss library
if (engine_type == static_cast<int>(engine::EngineType::FAISS_IVFSQ8H)) {
std::string msg = "Unsupported index type: " + std::to_string(index_type);
......
......@@ -348,7 +348,6 @@ TEST_F(DBTest, SEARCH_TEST) {
ASSERT_TRUE(stat.ok());
}
#ifdef CUSTOMIZATION
#ifdef MILVUS_GPU_VERSION
index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8H;
db_->CreateIndex(TABLE_NAME, index); // wait until build index finish
......@@ -360,7 +359,6 @@ TEST_F(DBTest, SEARCH_TEST) {
stat = db_->Query(dummy_context_, TABLE_NAME, tags, k, json_params, xq, result_ids, result_distances);
ASSERT_TRUE(stat.ok());
}
#endif
#endif
{ // search by specify index file
......@@ -411,7 +409,6 @@ TEST_F(DBTest, SEARCH_TEST) {
}
#endif
#ifdef CUSTOMIZATION
#ifdef MILVUS_GPU_VERSION
// test FAISS_IVFSQ8H optimizer
index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8H;
......@@ -441,7 +438,6 @@ TEST_F(DBTest, SEARCH_TEST) {
ASSERT_TRUE(stat.ok());
}
#endif
#endif
}
TEST_F(DBTest, PRELOADTABLE_TEST) {
......@@ -747,12 +743,10 @@ TEST_F(DBTest, INDEX_TEST) {
ASSERT_FALSE(stat.ok());
fiu_disable("DBImpl.UpdateTableIndexRecursively.fail_update_table_index");
#ifdef CUSTOMIZATION
#ifdef MILVUS_GPU_VERSION
index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8H;
stat = db_->CreateIndex(table_info.table_id_, index);
ASSERT_TRUE(stat.ok());
#endif
#endif
milvus::engine::TableIndex index_out;
......
......@@ -413,7 +413,7 @@ TEST(ValidationUtilTest, VALIDATE_INDEX_TEST) {
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexType((int)milvus::engine::EngineType::INVALID).code(),
milvus::SERVER_INVALID_INDEX_TYPE);
for (int i = 1; i <= (int)milvus::engine::EngineType::MAX_VALUE; i++) {
#ifndef CUSTOMIZATION
#ifndef MILVUS_GPU_VERSION
if (i == (int)milvus::engine::EngineType::FAISS_IVFSQ8H) {
ASSERT_NE(milvus::server::ValidationUtil::ValidateTableIndexType(i).code(), milvus::SERVER_SUCCESS);
continue;
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册