提交 86ce25ff 编写于 作者: Y Yu Kun

Merge remote-tracking branch 'upstream/branch-0.5.0' into branch-0.5.0-yk


Former-commit-id: 42e7261dba12f219ba620dfc35f6d7bbf554298b
......@@ -51,6 +51,7 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-600 - Reconstruct unittest code
- MS-602 - Remove zilliz namespace
- MS-610 - Change error code base value from hex to decimal
- MS-635 - Add compile option to support customized faiss
# Milvus 0.4.0 (2019-09-12)
......
......@@ -125,6 +125,10 @@ set(MILVUS_SOURCE_DIR ${PROJECT_SOURCE_DIR})
set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR})
set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src)
if (CUSTOMIZATION)
add_definitions(-DCUSTOMIZATION)
endif (CUSTOMIZATION)
include(ExternalProject)
include(DefineOptions)
include(BuildUtils)
......
......@@ -9,9 +9,12 @@ DB_PATH="/opt/milvus"
PROFILING="OFF"
USE_JFROG_CACHE="OFF"
RUN_CPPLINT="OFF"
CUSTOMIZATION="ON"
CUDA_COMPILER=/usr/local/cuda/bin/nvcc
while getopts "p:d:t:ulrcgjh" arg
wget -q --method HEAD
while getopts "p:d:t:ulrcgjhx" arg
do
case $arg in
p)
......@@ -45,6 +48,9 @@ do
j)
USE_JFROG_CACHE="ON"
;;
x)
CUSTOMIZATION="OFF"
;;
h) # help
echo "
......@@ -87,6 +93,7 @@ CMAKE_CMD="cmake \
-DMILVUS_DB_PATH=${DB_PATH} \
-DMILVUS_ENABLE_PROFILING=${PROFILING} \
-DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
-DCUSTOMIZATION=${CUSTOMIZATION} \
../"
echo ${CMAKE_CMD}
${CMAKE_CMD}
......
......@@ -228,17 +228,24 @@ foreach(_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT})
set(${_LIB_NAME} "${_LIB_VERSION}")
endforeach()
if(DEFINED ENV{KNOWHERE_FAISS_URL})
set(FAISS_SOURCE_URL "$ENV{KNOWHERE_FAISS_URL}")
else()
if(CUSTOMIZATION)
set(FAISS_SOURCE_URL "http://192.168.1.105:6060/jinhai/faiss/-/archive/${FAISS_VERSION}/faiss-${FAISS_VERSION}.tar.gz")
message(STATUS "FAISS URL = ${FAISS_SOURCE_URL}")
# set(FAISS_MD5 "a589663865a8558205533c8ac414278c")
# set(FAISS_MD5 "57da9c4f599cc8fa4260488b1c96e1cc") # commit-id 6dbdf75987c34a2c853bd172ea0d384feea8358c branch-0.2.0
# set(FAISS_MD5 "21deb1c708490ca40ecb899122c01403") # commit-id 643e48f479637fd947e7b93fa4ca72b38ecc9a39 branch-0.2.0
# set(FAISS_MD5 "072db398351cca6e88f52d743bbb9fa0") # commit-id 3a2344d04744166af41ef1a74449d68a315bfe17 branch-0.2.1
set(FAISS_MD5 "94988b7bdac4eb82a9575c702a3f2df3") # commit-id 1407526b31cad26f98ceca8dddaface8f18c4c19 branch-0.2.1
execute_process(COMMAND wget -q --method HEAD ${FAISS_SOURCE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote cache file ${FAISS_SOURCE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
set(FAISS_SOURCE_URL "https://github.com/facebookresearch/faiss/archive/v1.5.3.tar.gz")
endif()
else()
set(FAISS_SOURCE_URL "https://github.com/facebookresearch/faiss/archive/v1.5.3.tar.gz")
set(FAISS_MD5 "0bc12737b23def156f6a1eb782050135")
endif()
# set(FAISS_MD5 "a589663865a8558205533c8ac414278c")
# set(FAISS_MD5 "57da9c4f599cc8fa4260488b1c96e1cc") # commit-id 6dbdf75987c34a2c853bd172ea0d384feea8358c branch-0.2.0
# set(FAISS_MD5 "21deb1c708490ca40ecb899122c01403") # commit-id 643e48f479637fd947e7b93fa4ca72b38ecc9a39 branch-0.2.0
# set(FAISS_MD5 "072db398351cca6e88f52d743bbb9fa0") # commit-id 3a2344d04744166af41ef1a74449d68a315bfe17 branch-0.2.1
set(FAISS_MD5 "5af237d77947ee632f169bcb36feee2b") # commit-id 2c8affd0da60354e4322fa4c0224519e7912b9c4 branch-0.2.1
message(STATUS "FAISS URL = ${FAISS_SOURCE_URL}")
if(DEFINED ENV{KNOWHERE_ARROW_URL})
set(ARROW_SOURCE_URL "$ENV{KNOWHERE_ARROW_URL}")
......
......@@ -64,17 +64,13 @@ FaissBaseIndex::LoadImpl(const BinarySet& index_binary) {
void
FaissBaseIndex::SealImpl() {
// TODO(linxj): enable
//#ifdef ZILLIZ_FAISS
#ifdef CUSTOMIZATION
faiss::Index* index = index_.get();
auto idx = dynamic_cast<faiss::IndexIVF*>(index);
if (idx != nullptr) {
idx->to_readonly();
}
// else {
// KNOHWERE_ERROR_MSG("Seal failed");
//}
//#endif
#endif
}
} // namespace knowhere
......@@ -55,10 +55,15 @@ VectorIndexPtr
IVFSQ::CopyCpuToGpu(const int64_t& device_id, const Config& config) {
if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)) {
ResScope rs(res, device_id, false);
#ifdef CUSTOMIZATION
faiss::gpu::GpuClonerOptions option;
option.allInGpu = true;
auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, index_.get(), &option);
#else
auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, index_.get());
#endif
std::shared_ptr<faiss::Index> device_index;
device_index.reset(gpu_index);
......
......@@ -25,6 +25,7 @@
namespace knowhere {
#ifdef CUSTOMIZATION
IndexModelPtr
IVFSQHybrid::Train(const DatasetPtr& dataset, const Config& config) {
auto build_cfg = std::dynamic_pointer_cast<IVFSQCfg>(config);
......@@ -221,4 +222,50 @@ FaissIVFQuantizer::~FaissIVFQuantizer() {
// else do nothing
}
#else
QuantizerPtr
IVFSQHybrid::LoadQuantizer(const Config& conf) {
return knowhere::QuantizerPtr();
}
void
IVFSQHybrid::SetQuantizer(const QuantizerPtr& q) {
}
void
IVFSQHybrid::UnsetQuantizer() {
}
void
IVFSQHybrid::LoadData(const knowhere::QuantizerPtr& q, const Config& conf) {
}
IndexModelPtr
IVFSQHybrid::Train(const DatasetPtr& dataset, const Config& config) {
return GPUIVFSQ::Train(dataset, config);
}
VectorIndexPtr
IVFSQHybrid::CopyGpuToCpu(const Config& config) {
return GPUIVFSQ::CopyGpuToCpu(config);
}
VectorIndexPtr
IVFSQHybrid::CopyCpuToGpu(const int64_t& device_id, const Config& config) {
return IVF::CopyCpuToGpu(device_id, config);
}
void
IVFSQHybrid::search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels,
const Config& cfg) {
GPUIVF::search_impl(n, data, k, distances, labels, cfg);
}
void
IVFSQHybrid::LoadImpl(const BinarySet& index_binary) {
GPUIVF::LoadImpl(index_binary);
}
#endif
} // namespace knowhere
......@@ -25,12 +25,14 @@
namespace knowhere {
#ifdef CUSTOMIZATION
struct FaissIVFQuantizer : public Quantizer {
faiss::gpu::GpuIndexFlat* quantizer = nullptr;
~FaissIVFQuantizer() override;
};
using FaissIVFQuantizerPtr = std::shared_ptr<FaissIVFQuantizer>;
#endif
class IVFSQHybrid : public GPUIVFSQ {
public:
......
......@@ -70,7 +70,6 @@ enum class ParameterType {
ivf,
ivfpq,
ivfsq,
ivfsqhybrid,
nsg,
};
......@@ -104,7 +103,7 @@ class ParamGenerator {
tempconf->nbits = 8;
tempconf->metric_type = knowhere::METRICTYPE::L2;
return tempconf;
} else if (type == ParameterType::ivfsq || type == ParameterType::ivfsqhybrid) {
} else if (type == ParameterType::ivfsq) {
auto tempconf = std::make_shared<knowhere::IVFSQCfg>();
tempconf->d = DIM;
tempconf->gpu_id = device_id;
......@@ -158,8 +157,10 @@ INSTANTIATE_TEST_CASE_P(IVFParameters, IVFTest,
// std::make_tuple("IVFPQ", ParameterType::ivfpq),
// std::make_tuple("GPUIVFPQ", ParameterType::ivfpq),
std::make_tuple("IVFSQ", ParameterType::ivfsq),
std::make_tuple("GPUIVFSQ", ParameterType::ivfsq),
std::make_tuple("IVFSQHybrid", ParameterType::ivfsqhybrid)));
#ifdef CUSTOMIZATION
std::make_tuple("IVFSQHybrid", ParameterType::ivfsq),
#endif
std::make_tuple("GPUIVFSQ", ParameterType::ivfsq)));
void
AssertAnns(const knowhere::DatasetPtr& result, const int& nq, const int& k) {
......@@ -558,6 +559,7 @@ TEST_F(GPURESTEST, gpu_ivf_resource_test) {
}
}
#ifdef CUSTOMIZATION
TEST_F(GPURESTEST, gpuivfsq) {
{
// knowhere gpu ivfsq
......@@ -629,6 +631,7 @@ TEST_F(GPURESTEST, gpuivfsq) {
delete search_idx;
}
}
#endif
TEST_F(GPURESTEST, copyandsearch) {
// search and copy at the same time
......
......@@ -17,8 +17,8 @@
#include "scheduler/TaskTable.h"
#include "Utils.h"
#include "utils/Log.h"
#include "event/TaskTableUpdatedEvent.h"
#include "utils/Log.h"
#include <ctime>
#include <sstream>
......@@ -165,7 +165,8 @@ TaskTable::PickToLoad(uint64_t limit) {
}
if (table_[j]->state == TaskTableItemState::LOADED) {
++count;
if (count > 2) return std::vector<uint64_t >();
if (count > 2)
return std::vector<uint64_t>();
}
}
......
......@@ -17,12 +17,12 @@
#pragma once
#include "VecIndex.h"
#include "knowhere/index/vector_index/VectorIndex.h"
#include <memory>
#include <utility>
#include "VecIndex.h"
#include "knowhere/index/vector_index/VectorIndex.h"
namespace milvus {
namespace engine {
......
......@@ -143,10 +143,12 @@ GetVecIndexFactory(const IndexType& type, const Config& cfg) {
index = std::make_shared<knowhere::GPUIVFSQ>(gpu_device);
break;
}
#ifdef CUSTOMIZATION
case IndexType::FAISS_IVFSQ8_HYBRID: {
index = std::make_shared<knowhere::IVFSQHybrid>(gpu_device);
return std::make_shared<IVFHybridIndex>(index, IndexType::FAISS_IVFSQ8_HYBRID);
}
#endif
case IndexType::NSG_MIX: { // TODO(linxj): bug.
index = std::make_shared<knowhere::NSG>(gpu_device);
break;
......@@ -159,6 +161,9 @@ GetVecIndexFactory(const IndexType& type, const Config& cfg) {
VecIndexPtr
LoadVecIndex(const IndexType& index_type, const knowhere::BinarySet& index_binary) {
auto index = GetVecIndexFactory(index_type);
if (index == nullptr)
return nullptr;
// else
index->Load(index_binary);
return index;
}
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册