diff --git a/CHANGELOG.md b/CHANGELOG.md index d5f531e692796665c22fd89d160781eef3255ebb..48d1c5b6b1a2132cd012d4084b85a26f372b0fd2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,11 @@ Please mark all change in change log and use the ticket from JIRA. - \#440 - Query API in customization still uses old version - \#440 - Server cannot startup with gpu_resource_config.enable=false in GPU version - \#458 - Index data is not compatible between 0.5 and 0.6 +- \#465 - Server hang caused by searching with nsg index - \#486 - gpu no usage during index building +- \#509 - IVF_PQ index build trapped into dead loop caused by invalid params +- \#513 - Unittest DELETE_BY_RANGE sometimes failed +- \#527 - faiss benchmark not compatible with faiss 1.6.0 ## Feature - \#12 - Pure CPU version for Milvus diff --git a/README.md b/README.md index 110f0b6c1eadfd0a511f96c65a8acbe8b15d0aad..c06074277c58b8d9927422a30ffc34e59f4da03b 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ [![codebeat badge](https://codebeat.co/badges/e030a4f6-b126-4475-a938-4723d54ec3a7?style=plastic)](https://codebeat.co/projects/github-com-jinhai-cn-milvus-master) ![Release](https://img.shields.io/badge/release-v0.5.3-yellowgreen) ![Release_date](https://img.shields.io/badge/release%20date-November-yellowgreen) +[![codecov](https://codecov.io/gh/milvus-io/milvus/branch/master/graph/badge.svg)](https://codecov.io/gh/milvus-io/milvus) [中文版](README_CN.md) | [日本語版](README_JP.md) diff --git a/README_CN.md b/README_CN.md index b101ea05700caa913679755a2d127ffd31c0ae78..1c554ac0d851fc8979cdb54f0c46b827058ee54f 100644 --- a/README_CN.md +++ b/README_CN.md @@ -6,6 +6,8 @@ [![codebeat badge](https://codebeat.co/badges/e030a4f6-b126-4475-a938-4723d54ec3a7?style=plastic)](https://codebeat.co/projects/github-com-jinhai-cn-milvus-master) ![Release](https://img.shields.io/badge/release-v0.5.3-yellowgreen) ![Release_date](https://img.shields.io/badge/release_date-October-yellowgreen) +[![codecov](https://codecov.io/gh/milvus-io/milvus/branch/master/graph/badge.svg)](https://codecov.io/gh/milvus-io/milvus) + # 欢迎来到 Milvus diff --git a/ci/jenkins/step/package.groovy b/ci/jenkins/step/package.groovy index f0444084f8c614be93ff14ab9717a4c5fee441f7..35d2d8cfdf6be370ed6cb09d454a30ad52888708 100644 --- a/ci/jenkins/step/package.groovy +++ b/ci/jenkins/step/package.groovy @@ -4,6 +4,7 @@ timeout(time: 5, unit: 'MINUTES') { sh "./yaml_processor.py merge -f /opt/milvus/conf/server_config.yaml -m ../yaml/update_server_config.yaml -i && rm /opt/milvus/conf/server_config.yaml.bak" sh "sed -i 's/\\/tmp\\/milvus/\\/opt\\/milvus/g' /opt/milvus/conf/log_config.conf" } + sh "rm -rf /opt/milvus/unittest" sh "tar -zcvf ./${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz -C /opt/ milvus" withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) { def uploadStatus = sh(returnStatus: true, script: "curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -T ./${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz ${params.JFROG_ARTFACTORY_URL}/milvus/package/${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz") diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 671330c192e74351547e445e557e9f3d78861489..ba391c2d6bcd715caa41ed996515d21924e72329 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -31,9 +31,21 @@ GET_CURRENT_TIME(BUILD_TIME) string(REGEX REPLACE "\n" "" BUILD_TIME ${BUILD_TIME}) message(STATUS "Build time = ${BUILD_TIME}") +if (NOT DEFINED CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build.") +endif () + +set (GIT_BRANCH_NAME_REGEX "[0-9]+\\.[0-9]+\\.[0-9]") + MACRO(GET_GIT_BRANCH_NAME GIT_BRANCH_NAME) execute_process(COMMAND sh "-c" "git log --decorate | head -n 1 | sed 's/.*(\\(.*\\))/\\1/' | sed 's/.* \\(.*\\),.*/\\1/' | sed 's=[a-zA-Z]*\/==g'" OUTPUT_VARIABLE ${GIT_BRANCH_NAME}) + if(NOT GIT_BRANCH_NAME MATCHES "${GIT_BRANCH_NAME_REGEX}") + execute_process(COMMAND "git" rev-parse --abbrev-ref HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME}) + endif () + if(NOT GIT_BRANCH_NAME MATCHES "${GIT_BRANCH_NAME_REGEX}") + execute_process(COMMAND "git" symbolic-ref --short -q HEAD HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME}) + endif () ENDMACRO(GET_GIT_BRANCH_NAME) GET_GIT_BRANCH_NAME(GIT_BRANCH_NAME) @@ -43,7 +55,7 @@ if (NOT GIT_BRANCH_NAME STREQUAL "") endif () set(MILVUS_VERSION "${GIT_BRANCH_NAME}") -string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]" MILVUS_VERSION "${MILVUS_VERSION}") +string(REGEX MATCH "${GIT_BRANCH_NAME_REGEX}" MILVUS_VERSION "${MILVUS_VERSION}") if (CMAKE_BUILD_TYPE STREQUAL "Release") set(BUILD_TYPE "Release") diff --git a/core/build.sh b/core/build.sh index 3afb5d1b37a2af0f1d321caed7ebc8e1434d43a6..badcc5a032fbcbb8ce63a3a6fcd3c1f0c5f1e8dd 100755 --- a/core/build.sh +++ b/core/build.sh @@ -56,7 +56,7 @@ while getopts "p:d:t:f:ulrcgjhxzme" arg; do USE_JFROG_CACHE="ON" ;; x) - CUSTOMIZATION="OFF" # force use ori faiss + CUSTOMIZATION="ON" ;; g) GPU_VERSION="ON" diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 51ea665064d89c4447e3f2c77fe1aa57d2d1a1c7..1c18c224096d5ead5ce82c56457742247c1d3f47 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -105,7 +105,8 @@ DBImpl::Stop() { shutting_down_.store(true, std::memory_order_release); // makesure all memory data serialized - MemSerialize(); + std::set sync_table_ids; + SyncMemData(sync_table_ids); // wait compaction/buildindex finish bg_timer_thread_.join(); @@ -329,7 +330,10 @@ DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) { return SHUTDOWN_ERROR; } - Status status; + // serialize memory data + std::set sync_table_ids; + auto status = SyncMemData(sync_table_ids); + { std::unique_lock lock(build_index_mutex_); @@ -588,12 +592,12 @@ DBImpl::StartMetricTask() { } Status -DBImpl::MemSerialize() { +DBImpl::SyncMemData(std::set& sync_table_ids) { std::lock_guard lck(mem_serialize_mutex_); std::set temp_table_ids; mem_mgr_->Serialize(temp_table_ids); for (auto& id : temp_table_ids) { - compact_table_ids_.insert(id); + sync_table_ids.insert(id); } if (!temp_table_ids.empty()) { @@ -612,7 +616,7 @@ DBImpl::StartCompactionTask() { } // serialize memory data - MemSerialize(); + SyncMemData(compact_table_ids_); // compactiong has been finished? { diff --git a/core/src/db/DBImpl.h b/core/src/db/DBImpl.h index bff56efded8614ee4479e7072e9afcba453df846..82a5d3096b52939e37e46fff3cd07d74a76a6a27 100644 --- a/core/src/db/DBImpl.h +++ b/core/src/db/DBImpl.h @@ -150,7 +150,7 @@ class DBImpl : public DB { BackgroundBuildIndex(); Status - MemSerialize(); + SyncMemData(std::set& sync_table_ids); Status GetFilesToBuildIndex(const std::string& table_id, const std::vector& file_types, diff --git a/core/src/index/cmake/BuildUtilsCore.cmake b/core/src/index/cmake/BuildUtilsCore.cmake index 1bf6ba2af423d5f8cb5bd360a2cefa1fa6d23638..2261ce3e29768bf15082d0b2323bd16b7e645d81 100644 --- a/core/src/index/cmake/BuildUtilsCore.cmake +++ b/core/src/index/cmake/BuildUtilsCore.cmake @@ -74,7 +74,7 @@ function(ExternalProject_Use_Cache project_name package_file install_path) ${CMAKE_COMMAND} -E echo "Extracting ${package_file} to ${install_path}" COMMAND - ${CMAKE_COMMAND} -E tar xzvf ${package_file} ${install_path} + ${CMAKE_COMMAND} -E tar xzf ${package_file} ${install_path} WORKING_DIRECTORY ${INDEX_BINARY_DIR} ) diff --git a/core/src/index/knowhere/knowhere/common/Timer.cpp b/core/src/index/knowhere/knowhere/common/Timer.cpp index fefe1e1705bb32a7b739af87ad3e1e5360ebdf31..141e1ceed6a20d61c9dfeb31a517b111e8ec45e6 100644 --- a/core/src/index/knowhere/knowhere/common/Timer.cpp +++ b/core/src/index/knowhere/knowhere/common/Timer.cpp @@ -17,6 +17,7 @@ #include // TODO(linxj): using Log instead +#include "knowhere/common/Log.h" #include "knowhere/common/Timer.h" namespace knowhere { @@ -51,30 +52,18 @@ TimeRecorder::PrintTimeRecord(const std::string& msg, double span) { std::cout << str_log << std::endl; break; } - // case 1: { - // SERVER_LOG_DEBUG << str_log; - // break; - //} + case 1: { + KNOWHERE_LOG_DEBUG << str_log; + break; + } // case 2: { - // SERVER_LOG_INFO << str_log; - // break; - //} + // KNOWHERE_LOG_TRACE << str_log; + // break; + // } // case 3: { - // SERVER_LOG_WARNING << str_log; - // break; - //} - // case 4: { - // SERVER_LOG_ERROR << str_log; - // break; - //} - // case 5: { - // SERVER_LOG_FATAL << str_log; - // break; - //} - // default: { - // SERVER_LOG_INFO << str_log; - // break; - //} + // KNOWHERE_LOG_WARNING << str_log; + // break; + // } } } diff --git a/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp b/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp index bdf538c20479ec7e3d4d039fb3b24ef0ecf22e03..b4e00e57b7b6b0420815b016fcbceb7da12ac1d6 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/nsg/NSG.cpp @@ -718,29 +718,38 @@ NsgIndex::Search(const float* query, const unsigned& nq, const unsigned& dim, co int64_t* ids, SearchParams& params) { std::vector> resset(nq); - TimeRecorder rc("search"); - if (nq == 1) { + params.search_length = k; + TimeRecorder rc("NsgIndex::search", 1); + // TODO(linxj): when to use openmp + if (nq <= 4) { GetNeighbors(query, resset[0], nsg, ¶ms); } else { -//#pragma omp parallel for schedule(dynamic, 50) #pragma omp parallel for for (unsigned int i = 0; i < nq; ++i) { - // TODO(linxj): when to use openmp auto single_query = query + i * dim; GetNeighbors(single_query, resset[i], nsg, ¶ms); } } - rc.ElapseFromBegin("cost"); - + rc.RecordSection("search"); for (unsigned int i = 0; i < nq; ++i) { - for (unsigned int j = 0; j < k; ++j) { - // ids[i * k + j] = resset[i][j].id; - - // Fix(linxj): bug, reset[i][j] out of range - ids[i * k + j] = ids_[resset[i][j].id]; - dist[i * k + j] = resset[i][j].distance; + int64_t var = resset[i].size() - k; + if (var >= 0) { + for (unsigned int j = 0; j < k; ++j) { + ids[i * k + j] = ids_[resset[i][j].id]; + dist[i * k + j] = resset[i][j].distance; + } + } else { + for (unsigned int j = 0; j < resset[i].size(); ++j) { + ids[i * k + j] = ids_[resset[i][j].id]; + dist[i * k + j] = resset[i][j].distance; + } + for (unsigned int j = resset[i].size(); j < k; ++j) { + ids[i * k + j] = -1; + dist[i * k + j] = -1; + } } } + rc.RecordSection("merge"); //>> Debug: test single insert // int x_0 = resset[0].size(); diff --git a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp index ce5aa4bafab7045a1567605beed07727a4102601..0d49004621c4da0833cf115f8d003c5a11d851a2 100644 --- a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp +++ b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#define USE_FAISS_V0_2_1 0 + #include #include @@ -26,30 +28,29 @@ #include #include -#define USE_FAISS_V1_5_3 0 +#include +#include +#include +#include +#include +#include -#if USE_FAISS_V1_5_3 +#if USE_FAISS_V0_2_1 #include #include #include #include #include - #else #include #include #include #endif -#include -#include -#include -#include -#include -#include - #ifdef CUSTOMIZATION #include +#else +#include #endif /***************************************************** @@ -295,10 +296,12 @@ load_base_data(faiss::Index*& index, const std::string& ann_test_name, const std cpu_index = faiss::gpu::index_gpu_to_cpu(gpu_index); delete gpu_index; +#ifdef CUSTOMIZATION faiss::IndexIVF* cpu_ivf_index = dynamic_cast(cpu_index); if (cpu_ivf_index != nullptr) { cpu_ivf_index->to_readonly(); } +#endif printf("[%.3f s] Writing index file: %s\n", elapsed() - t0, index_file_name.c_str()); faiss::write_index(cpu_index, index_file_name.c_str()); @@ -374,13 +377,15 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key faiss::Index *gpu_index, *index; if (query_mode != MODE_CPU) { faiss::gpu::GpuClonerOptions option; +#ifdef CUSTOMIZATION option.allInGpu = true; faiss::IndexComposition index_composition; index_composition.index = cpu_index; index_composition.quantizer = nullptr; - +#endif switch (query_mode) { +#ifdef CUSTOMIZATION case MODE_MIX: { index_composition.mode = 1; // 0: all data, 1: copy quantizer, 2: copy data @@ -403,7 +408,9 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key index = cpu_index; break; } +#endif case MODE_GPU: +#ifdef CUSTOMIZATION index_composition.mode = 0; // 0: all data, 1: copy quantizer, 2: copy data // warm up the transmission @@ -412,6 +419,14 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key copy_time = elapsed(); gpu_index = faiss::gpu::index_cpu_to_gpu(&res, GPU_DEVICE_IDX, &index_composition, &option); +#else + // warm up the transmission + gpu_index = faiss::gpu::index_cpu_to_gpu(&res, GPU_DEVICE_IDX, cpu_index, &option); + delete gpu_index; + + copy_time = elapsed(); + gpu_index = faiss::gpu::index_cpu_to_gpu(&res, GPU_DEVICE_IDX, cpu_index, &option); +#endif copy_time = elapsed() - copy_time; printf("[%.3f s] Copy data completed, cost %f s\n", elapsed() - t0, copy_time); diff --git a/core/src/wrapper/ConfAdapter.cpp b/core/src/wrapper/ConfAdapter.cpp index aa4b3c12b8a98ff1e1ac04ef815fdcd24e7b6b77..d49747d8f4ccd98c98385af7519312783d87fe4e 100644 --- a/core/src/wrapper/ConfAdapter.cpp +++ b/core/src/wrapper/ConfAdapter.cpp @@ -22,6 +22,7 @@ #include #include +#include // TODO(lxj): add conf checker @@ -129,17 +130,35 @@ IVFPQConfAdapter::Match(const TempMetaConf& metaconf) { conf->metric_type = metaconf.metric_type; conf->gpu_id = metaconf.gpu_id; conf->nbits = 8; + MatchBase(conf); - if (!(conf->d % 4)) - conf->m = conf->d / 4; // compression radio = 16 - else if (!(conf->d % 2)) - conf->m = conf->d / 2; // compression radio = 8 - else if (!(conf->d % 3)) - conf->m = conf->d / 3; // compression radio = 12 - else - conf->m = conf->d; // same as SQ8, compression radio = 4 + /* + * Faiss 1.6 + * Only 1, 2, 3, 4, 6, 8, 10, 12, 16, 20, 24, 28, 32 dims per sub-quantizer are currently supporte with + * no precomputed codes. Precomputed codes supports any number of dimensions, but will involve memory overheads. + */ + static std::vector support_dim_per_subquantizer{32, 28, 24, 20, 16, 12, 10, 8, 6, 4, 3, 2, 1}; + static std::vector support_subquantizer{96, 64, 56, 48, 40, 32, 28, 24, 20, 16, 12, 8, 4, 3, 2, 1}; + std::vector resset; + for (const auto& dimperquantizer : support_dim_per_subquantizer) { + if (!(conf->d % dimperquantizer)) { + auto subquantzier_num = conf->d / dimperquantizer; + auto finder = std::find(support_subquantizer.begin(), support_subquantizer.end(), subquantzier_num); + if (finder != support_subquantizer.end()) { + resset.push_back(subquantzier_num); + } + } + } - MatchBase(conf); + if (resset.empty()) { + // todo(linxj): throw exception here. + return nullptr; + } + static int64_t compression_level = 1; // 1:low, 2:high + if (compression_level == 1) { + conf->m = resset[int(resset.size() / 2)]; + WRAPPER_LOG_DEBUG << "PQ m = " << conf->m << ", compression radio = " << conf->d / conf->m * 4; + } return conf; } diff --git a/tests/milvus_python_test/test_index.py b/tests/milvus_python_test/test_index.py index 108563691fab08594477f45fa3e25eef607a0fa5..b253cf02a36b34f990ff9db74c3f2512efdcf752 100644 --- a/tests/milvus_python_test/test_index.py +++ b/tests/milvus_python_test/test_index.py @@ -1208,17 +1208,9 @@ class TestCreateIndexParamsInvalid(object): nlist = index_params["nlist"] logging.getLogger().info(index_params) status, ids = connect.add_vectors(table, vectors) - # if not isinstance(index_type, int) or not isinstance(nlist, int): - try: + if (not index_type) or (not nlist) or (not isinstance(index_type, IndexType)) or (not isinstance(nlist, int)): + with pytest.raises(Exception) as e: + status = connect.create_index(table, index_params) + else: status = connect.create_index(table, index_params) assert not status.OK() - # no exception raised & status is OK. unexpected. - assert False - except (Exception, ): - pass - # with pytest.raises(Exception) as e: - # status = connect.create_index(table, index_params) - # assert not status.OK() - # else: - # status = connect.create_index(table, index_params) - # assert not status.OK() diff --git a/tests/milvus_python_test/utils.py b/tests/milvus_python_test/utils.py index 6cb63b36c4c2e963ed14a58e8d927f4c3f07e5fa..6f7c81d13502090576b7d839ca3806d54c4e2bcc 100644 --- a/tests/milvus_python_test/utils.py +++ b/tests/milvus_python_test/utils.py @@ -58,7 +58,7 @@ def gen_invalid_ips(): # "255.255.0.0", # "255.255.255.0", # "255.255.255.255", - # "127.0.0", + "127.0.0", # "123.0.0.2", "12-s", " ", @@ -69,7 +69,7 @@ def gen_invalid_ips(): "\n", "\t", "中文", - "a".join("a" for i in range(256)) + "a".join("a" for _ in range(256)) ] return ips @@ -263,9 +263,7 @@ def gen_invalid_index_types(): def gen_invalid_nlists(): nlists = [ - 0, -1, - 1000000000000001, # None, [1,2,3], (1,2), @@ -549,4 +547,4 @@ if __name__ == "__main__": p.join() time.sleep(3) status, count = milvus.get_table_row_count(table) - assert count == process_num * loop_num \ No newline at end of file + assert count == process_num * loop_num