Skip to content

milvus
milvus

qq_34331598 / milvus
与 Fork 源项目一致

Fork自 milvus / milvus

通知 2
Star 0
Fork 0
milvus
milvus

体验新版 GitCode，发现更多精彩内容 >>

未验证 提交 6675a9a4 编写于 作者: B BossZou 提交者： GitHub
浏览文件
操作

(db/snapshot) Remove sqlite-orm (#3028) 

* Remove SqliteMetaImpl
Signed-off-by: Nyhz <413554850@qq.com>

* Remove sqlite-orm
Signed-off-by: Nyhz <413554850@qq.com>
上级 ccfca780
展开全部 隐藏空白更改
内联 并排
Showing with 164 addition and 2736 deletion
CHANGELOG.md
浏览文件 @ 6675a9a4
......@@ -67,6 +67,7 @@ Please mark all changes in change log and use the issue from GitHub
- \#2612 Move all APIs in utils into namespace milvus
- \#2653 Improve IVF search performance when NQ and nProbe are both large
- \#2675 Print out system memory size when report invalid cpu cache size
- \#2686 Remove dependency on sqlite_orm
- \#2828 Let Faiss not compile half float by default
- \#2841 Replace IndexType/EngineType/MetricType
- \#2858 Unify index name in db
......
core/cmake/DefineOptions.cmake
浏览文件 @ 6675a9a4
......@@ -68,8 +68,6 @@ define_option(MILVUS_WITH_PROMETHEUS "Build with PROMETHEUS library" ON)
define_option(MILVUS_WITH_SQLITE "Build with SQLite library" ON)
define_option(MILVUS_WITH_SQLITE_ORM "Build with SQLite ORM library" ON)
define_option(MILVUS_WITH_MYSQLPP "Build with MySQL++" ON)
define_option(MILVUS_WITH_YAMLCPP "Build with yaml-cpp library" ON)
......
core/cmake/ThirdPartyPackages.cmake
浏览文件 @ 6675a9a4
......@@ -15,7 +15,6 @@ set(MILVUS_THIRDPARTY_DEPENDENCIES
MySQLPP
Prometheus
SQLite
SQLite_ORM
yaml-cpp
libunwind
gperftools
......@@ -44,8 +43,6 @@ macro(build_dependency DEPENDENCY_NAME)
build_prometheus()
elseif ("${DEPENDENCY_NAME}" STREQUAL "SQLite")
build_sqlite()
elseif ("${DEPENDENCY_NAME}" STREQUAL "SQLite_ORM")
build_sqlite_orm()
elseif ("${DEPENDENCY_NAME}" STREQUAL "yaml-cpp")
build_yamlcpp()
elseif ("${DEPENDENCY_NAME}" STREQUAL "libunwind")
......@@ -261,14 +258,6 @@ else ()
"https://www.sqlite.org/2019/sqlite-autoconf-${SQLITE_VERSION}.tar.gz")
endif ()
if (DEFINED ENV{MILVUS_SQLITE_ORM_URL})
set(SQLITE_ORM_SOURCE_URLS "$ENV{MILVUS_SQLITE_ORM_URL}")
else ()
set(SQLITE_ORM_SOURCE_URLS
"https://github.com/fnc12/sqlite_orm/archive/${SQLITE_ORM_VERSION}.zip"
"https://gitee.com/quicksilver/sqlite_orm/repository/archive/${SQLITE_ORM_VERSION}.zip")
endif ()
if (DEFINED ENV{MILVUS_YAMLCPP_URL})
set(YAMLCPP_SOURCE_URL "$ENV{MILVUS_YAMLCPP_URL}")
else ()
......@@ -603,64 +592,6 @@ if (MILVUS_WITH_SQLITE)
link_directories(SYSTEM ${SQLITE_PREFIX}/lib/)
endif ()
# ----------------------------------------------------------------------
# SQLite_ORM
macro(build_sqlite_orm)
message(STATUS "Building SQLITE_ORM-${SQLITE_ORM_VERSION} from source")
set(SQLITE_ORM_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/sqlite_orm_ep-prefix")
set(SQLITE_ORM_TAR_NAME "${SQLITE_ORM_PREFIX}/sqlite_orm-${SQLITE_ORM_VERSION}.tar.gz")
set(SQLITE_ORM_INCLUDE_DIR "${SQLITE_ORM_PREFIX}/sqlite_orm-${SQLITE_ORM_VERSION}/include/sqlite_orm")
if (NOT EXISTS ${SQLITE_ORM_INCLUDE_DIR})
file(MAKE_DIRECTORY ${SQLITE_ORM_PREFIX})
set(IS_EXIST_FILE FALSE)
foreach(url ${SQLITE_ORM_SOURCE_URLS})
file(DOWNLOAD ${url}
${SQLITE_ORM_TAR_NAME}
TIMEOUT 60
STATUS status
LOG log)
list(GET status 0 status_code)
list(GET status 1 status_string)
if(status_code EQUAL 0)
message(STATUS "Downloading SQLITE_ORM ... done")
set(IS_EXIST_FILE TRUE)
break()
else()
string(APPEND logFailedURLs "error: downloading '${url}' failed
status_code: ${status_code}
status_string: ${status_string}
log:
--- LOG BEGIN ---
${log}
--- LOG END ---
"
)
endif()
endforeach()
if(IS_EXIST_FILE STREQUAL "FALSE")
message(FATAL_ERROR "Each download failed!
${logFailedURLs}
"
)
endif()
execute_process(COMMAND ${CMAKE_COMMAND} -E tar -xf ${SQLITE_ORM_TAR_NAME}
WORKING_DIRECTORY ${SQLITE_ORM_PREFIX})
endif ()
endmacro()
if (MILVUS_WITH_SQLITE_ORM)
resolve_dependency(SQLite_ORM)
include_directories(SYSTEM "${SQLITE_ORM_INCLUDE_DIR}")
endif ()
# ----------------------------------------------------------------------
# yaml-cpp
......
core/src/db/DBFactory.cpp
浏览文件 @ 6675a9a4
......@@ -10,17 +10,8 @@
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "db/DBFactory.h"
#include "DBImpl.h"
#include "db/DBImpl.h"
#include "meta/MetaFactory.h"
#include "meta/MySQLMetaImpl.h"
#include "meta/SqliteMetaImpl.h"
#include "utils/Exception.h"
#include <stdlib.h>
#include <time.h>
#include <cstdlib>
#include <sstream>
#include <string>
namespace milvus {
namespace engine {
......
core/src/db/Options.cpp
浏览文件 @ 6675a9a4
......@@ -46,7 +46,7 @@ ArchiveConf::ParseCritirias(const std::string& criterias) {
boost::algorithm::split(tokens, criterias, boost::is_any_of(";"));
fiu_do_on("ArchiveConf.ParseCritirias.empty_tokens", tokens.clear());
if (tokens.size() == 0) {
if (tokens.empty()) {
return;
}
......
core/src/db/meta/MetaFactory.cpp
浏览文件 @ 6675a9a4
......@@ -10,22 +10,17 @@
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "db/meta/MetaFactory.h"
#include "MySQLMetaImpl.h"
#include "SqliteMetaImpl.h"
#include "db/Utils.h"
#include "utils/Exception.h"
#include "utils/Log.h"
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <cstdlib>
#include <memory>
#include <sstream>
#include <string>
namespace milvus {
namespace engine {
namespace milvus::engine {
DBMetaOptions
MetaFactory::BuildOption(const std::string& path) {
......@@ -43,8 +38,8 @@ MetaFactory::BuildOption(const std::string& path) {
return meta;
}
meta::MetaPtr
MetaFactory::Build(const DBMetaOptions& meta_options, const int& mode) {
meta::MetaAdapterPtr
MetaFactory::Build(const DBMetaOptions& meta_options) {
std::string uri = meta_options.backend_uri_;
utils::MetaUriInfo uri_info;
......@@ -57,15 +52,23 @@ MetaFactory::Build(const DBMetaOptions& meta_options, const int& mode) {
if (strcasecmp(uri_info.dialect_.c_str(), "mysql") == 0) {
LOG_ENGINE_INFO_ << "Using MySQL";
return std::make_shared<meta::MySQLMetaImpl>(meta_options, mode);
/* options.backend_uri_ = "mysql://root:12345678@127.0.0.1:3307/milvus"; */
auto engine = std::make_shared<meta::MySqlEngine>(meta_options);
return std::make_shared<meta::MetaAdapter>(engine);
} else if (strcasecmp(uri_info.dialect_.c_str(), "mock") == 0) {
LOG_ENGINE_INFO_ << "Using Mock. Should only be used in test environment";
auto engine = std::make_shared<meta::MockEngine>();
return std::make_shared<meta::MetaAdapter>(engine);
} else if (strcasecmp(uri_info.dialect_.c_str(), "sqlite") == 0) {
LOG_ENGINE_INFO_ << "Using SQLite";
return std::make_shared<meta::SqliteMetaImpl>(meta_options);
LOG_ENGINE_INFO_ << "Using Sqlite";
DBMetaOptions options;
/* options.backend_uri_ = "mock://:@:/"; */
auto engine = std::make_shared<meta::SqliteEngine>(options);
return std::make_shared<meta::MetaAdapter>(engine);
} else {
LOG_ENGINE_ERROR_ << "Invalid dialect in URI: dialect = " << uri_info.dialect_;
throw InvalidArgumentException("URI dialect is not mysql / sqlite");
throw InvalidArgumentException("URI dialect is not mysql / sqlite / mock");
}
}
} // namespace engine
} // namespace milvus
} // namespace milvus::engine
core/src/db/meta/MetaFactory.h
浏览文件 @ 6675a9a4
......@@ -11,11 +11,11 @@
#pragma once
#include "Meta.h"
#include "db/Options.h"
#include <string>
#include "db/Options.h"
#include "db/meta/MetaAdapter.h"
namespace milvus {
namespace engine {
......@@ -24,8 +24,8 @@ class MetaFactory {
static DBMetaOptions
BuildOption(const std::string& path = "");
static meta::MetaPtr
Build(const DBMetaOptions& meta_options, const int& mode);
static meta::MetaAdapterPtr
Build(const DBMetaOptions& meta_options);
};
} // namespace engine
......
core/src/db/meta/MySQLMetaImpl.cpp
浏览文件 @ 6675a9a4
......@@ -2327,6 +2327,7 @@ MySQLMetaImpl::FilesByID(const std::vector<size_t>& ids, FilesHolder& files_hold
}
// TODO(myh): Support swap to cloud storage
// TODO(yhz): May use archive_conf_ in MySqlEngine
Status
MySQLMetaImpl::Archive() {
auto& criterias = options_.archive_conf_.GetCriterias();
......
core/src/db/meta/SqliteMetaImpl.cpp 已删除 100644 → 0
浏览文件 @ ccfca780
此差异已折叠。
core/src/db/meta/SqliteMetaImpl.h 已删除 100644 → 0
浏览文件 @ ccfca780
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <mutex>
#include <set>
#include <string>
#include <vector>
#include "Meta.h"
#include "db/Options.h"
namespace milvus {
namespace engine {
namespace meta {
auto
StoragePrototype(const std::string& path);
auto
CollectionPrototype(const std::string& path);
class SqliteMetaImpl : public Meta {
public:
explicit SqliteMetaImpl(const DBMetaOptions& options);
~SqliteMetaImpl();
Status
CreateCollection(CollectionSchema& collection_schema) override;
Status
DescribeCollection(CollectionSchema& collection_schema) override;
Status
HasCollection(const std::string& collection_id, bool& has_or_not, bool is_root = false) override;
Status
AllCollections(std::vector<CollectionSchema>& collection_schema_array, bool is_root = false) override;
Status
DropCollections(const std::vector<std::string>& collection_id_array) override;
Status
DeleteCollectionFiles(const std::vector<std::string>& collection_id_array) override;
Status
CreateCollectionFile(SegmentSchema& file_schema) override;
Status
GetCollectionFiles(const std::string& collection_id, const std::vector<size_t>& ids,
FilesHolder& files_holder) override;
Status
GetCollectionFilesBySegmentId(const std::string& segment_id, FilesHolder& files_holder) override;
Status
UpdateCollectionIndex(const std::string& collection_id, const CollectionIndex& index) override;
Status
UpdateCollectionFlag(const std::string& collection_id, int64_t flag) override;
Status
UpdateCollectionFlushLSN(const std::string& collection_id, uint64_t flush_lsn) override;
Status
GetCollectionFlushLSN(const std::string& collection_id, uint64_t& flush_lsn) override;
Status
UpdateCollectionFile(SegmentSchema& file_schema) override;
Status
UpdateCollectionFilesToIndex(const std::string& collection_id) override;
Status
UpdateCollectionFiles(SegmentsSchema& files) override;
Status
UpdateCollectionFilesRowCount(SegmentsSchema& files) override;
Status
DescribeCollectionIndex(const std::string& collection_id, CollectionIndex& index) override;
Status
DropCollectionIndex(const std::string& collection_id) override;
Status
CreatePartition(const std::string& collection_id, const std::string& partition_name, const std::string& tag,
uint64_t lsn) override;
Status
HasPartition(const std::string& collection_id, const std::string& tag, bool& has_or_not) override;
Status
DropPartition(const std::string& partition_name) override;
Status
ShowPartitions(const std::string& collection_id,
std::vector<meta::CollectionSchema>& partition_schema_array) override;
Status
GetPartitionName(const std::string& collection_id, const std::string& tag, std::string& partition_name) override;
Status
FilesToSearch(const std::string& collection_id, FilesHolder& files_holder) override;
Status
FilesToSearchEx(const std::string& root_collection, const std::set<std::string>& partition_id_array,
FilesHolder& files_holder) override;
Status
FilesToMerge(const std::string& collection_id, FilesHolder& files_holder) override;
Status
FilesToIndex(FilesHolder& files_holder) override;
Status
FilesByType(const std::string& collection_id, const std::vector<int>& file_types,
FilesHolder& files_holder) override;
Status
FilesByTypeEx(const std::vector<meta::CollectionSchema>& collections, const std::vector<int>& file_types,
FilesHolder& files_holder) override;
Status
FilesByID(const std::vector<size_t>& ids, FilesHolder& files_holder) override;
Status
Size(uint64_t& result) override;
Status
Archive() override;
Status
CleanUpShadowFiles() override;
Status
CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter = nullptr*/) override;
Status
DropAll() override;
Status
Count(const std::string& collection_id, uint64_t& result) override;
Status
SetGlobalLastLSN(uint64_t lsn) override;
Status
GetGlobalLastLSN(uint64_t& lsn) override;
Status
CreateHybridCollection(CollectionSchema& collection_schema, hybrid::FieldsSchema& fields_schema) override;
Status
DescribeHybridCollection(CollectionSchema& collection_schema, hybrid::FieldsSchema& fields_schema) override;
private:
Status
NextFileId(std::string& file_id);
Status
NextCollectionId(std::string& collection_id);
Status
DiscardFiles(int64_t to_discard_size);
void
ValidateMetaSchema();
Status
Initialize();
private:
const DBMetaOptions options_;
std::mutex meta_mutex_;
std::mutex genid_mutex_;
}; // DBMetaImpl
} // namespace meta
} // namespace engine
} // namespace milvus
core/src/db/snapshot/Store.h
浏览文件 @ 6675a9a4
......@@ -12,7 +12,7 @@
#pragma once
#include "db/Utils.h"
#include "db/meta/MetaAdapter.h"
#include "db/meta/MetaFactory.h"
#include "db/snapshot/ResourceContext.h"
#include "db/snapshot/ResourceTypes.h"
#include "db/snapshot/Resources.h"
......@@ -50,45 +50,17 @@ class Store : public std::enable_shared_from_this<Store> {
using Ptr = typename std::shared_ptr<Store>;
explicit Store(meta::MetaAdapterPtr adapter, const std::string& root_path)
: adapter_(adapter), root_path_(root_path) {
: adapter_(std::move(adapter)), root_path_(root_path) {
}
static Store::Ptr
Build(const std::string& uri, const std::string& root_path) {
utils::MetaUriInfo uri_info;
LOG_ENGINE_DEBUG_ << "MetaUri: " << uri << std::endl;
auto status = utils::ParseMetaUri(uri, uri_info);
if (!status.ok()) {
LOG_ENGINE_ERROR_ << "Wrong URI format: URI = " << uri;
throw InvalidArgumentException("Wrong URI format ");
}
DBMetaOptions options;
options.backend_uri_ = uri;
options.path_ = root_path;
if (strcasecmp(uri_info.dialect_.c_str(), "mysql") == 0) {
LOG_ENGINE_INFO_ << "Using MySQL";
DBMetaOptions options;
/* options.backend_uri_ = "mysql://root:12345678@127.0.0.1:3307/milvus"; */
options.backend_uri_ = uri;
auto engine = std::make_shared<meta::MySqlEngine>(options);
auto adapter = std::make_shared<meta::MetaAdapter>(engine);
return std::make_shared<Store>(adapter, root_path);
} else if (strcasecmp(uri_info.dialect_.c_str(), "mock") == 0) {
LOG_ENGINE_INFO_ << "Using Mock. Should only be used in test environment";
auto engine = std::make_shared<meta::MockEngine>();
auto adapter = std::make_shared<meta::MetaAdapter>(engine);
return std::make_shared<Store>(adapter, root_path);
} else if (strcasecmp(uri_info.dialect_.c_str(), "sqlite") == 0) {
LOG_ENGINE_INFO_ << "Using Sqlite";
DBMetaOptions options;
/* options.backend_uri_ = "mock://:@:/"; */
options.backend_uri_ = uri;
options.path_ = root_path;
auto engine = std::make_shared<meta::SqliteEngine>(options);
auto adapter = std::make_shared<meta::MetaAdapter>(engine);
return std::make_shared<Store>(adapter, root_path);
} else {
LOG_ENGINE_ERROR_ << "Invalid dialect in URI: dialect = " << uri_info.dialect_;
throw InvalidArgumentException("URI dialect is not mysql / sqlite / mock");
}
auto adapter = MetaFactory::Build(options);
return std::make_shared<Store>(adapter, root_path);
}
std::string
......
core/thirdparty/versions.txt
浏览文件 @ 6675a9a4
......@@ -3,7 +3,6 @@ GTEST_VERSION=1.8.1
MYSQLPP_VERSION=3.2.4
PROMETHEUS_VERSION=0.7.0
SQLITE_VERSION=3280000
SQLITE_ORM_VERSION=master
YAMLCPP_VERSION=0.6.2
LIBUNWIND_VERSION=1.3.1
GPERFTOOLS_VERSION=2.7
......
core/unittest/db/test_ss_task.cpp
浏览文件 @ 6675a9a4
......@@ -14,7 +14,6 @@
#include <gtest/gtest.h>
#include <opentracing/mocktracer/tracer.h>
//#include "db/meta/SqliteMetaImpl.h"
#include "db/DBFactory.h"
#include "scheduler/SchedInst.h"
#include "scheduler/job/BuildIndexJob.h"
......
core/unittest/scheduler/test_task.cpp
浏览文件 @ 6675a9a4
......@@ -16,7 +16,6 @@
#include <src/scheduler/SchedInst.h>
#include <src/scheduler/resource/CpuResource.h>
#include "db/meta/SqliteMetaImpl.h"
#include "db/DBFactory.h"
#include "scheduler/tasklabel/BroadcastLabel.h"
#include "scheduler/task/BuildIndexTask.h"
......@@ -48,136 +47,136 @@ TEST(TaskTest, INVALID_INDEX) {
build_task->Execute();
}
TEST(TaskTest, TEST_TASK) {
auto dummy_context = std::make_shared<milvus::server::Context>("dummy_request_id");
auto file = std::make_shared<SegmentSchema>();
file->index_params_ = "{ \"nlist\": 16384 }";
file->dimension_ = 64;
auto label = std::make_shared<BroadcastLabel>();
TestTask task(dummy_context, file, label);
task.Load(LoadType::CPU2GPU, 0);
auto th = std::thread([&]() {
task.Execute();
});
task.Wait();
if (th.joinable()) {
th.join();
}
static const char* CONFIG_PATH = "/tmp/milvus_test";
auto options = milvus::engine::DBFactory::BuildOption();
options.meta_.path_ = CONFIG_PATH;
options.meta_.backend_uri_ = "sqlite://:@:/";
options.insert_cache_immediately_ = true;
auto meta_ptr = std::make_shared<milvus::engine::meta::SqliteMetaImpl>(options.meta_);
file->collection_id_ = "111";
file->location_ = "/tmp/milvus_test/index_file1.txt";
auto build_index_job = std::make_shared<BuildIndexJob>(meta_ptr, options);
XBuildIndexTask build_index_task(file, label);
build_index_task.job_ = build_index_job;
build_index_task.Load(LoadType::TEST, 0);
fiu_init(0);
fiu_enable("XBuildIndexTask.Load.throw_std_exception", 1, NULL, 0);
build_index_task.Load(LoadType::TEST, 0);
fiu_disable("XBuildIndexTask.Load.throw_std_exception");
fiu_enable("XBuildIndexTask.Load.out_of_memory", 1, NULL, 0);
build_index_task.Load(LoadType::TEST, 0);
fiu_disable("XBuildIndexTask.Load.out_of_memory");
build_index_task.Execute();
// always enable 'create_table_success'
fiu_enable("XBuildIndexTask.Execute.create_table_success", 1, NULL, 0);
milvus::json json = {{"nlist", 16384}};
build_index_task.to_index_engine_ =
EngineFactory::Build(file->dimension_, file->location_, (EngineType)file->engine_type_,
(MetricType)file->metric_type_, json);
build_index_task.Execute();
fiu_enable("XBuildIndexTask.Execute.build_index_fail", 1, NULL, 0);
build_index_task.to_index_engine_ =
EngineFactory::Build(file->dimension_, file->location_, (EngineType)file->engine_type_,
(MetricType)file->metric_type_, json);
build_index_task.Execute();
fiu_disable("XBuildIndexTask.Execute.build_index_fail");
// always enable 'has_collection'
fiu_enable("XBuildIndexTask.Execute.has_collection", 1, NULL, 0);
build_index_task.to_index_engine_ =
EngineFactory::Build(file->dimension_, file->location_, (EngineType)file->engine_type_,
(MetricType)file->metric_type_, json);
build_index_task.Execute();
fiu_enable("XBuildIndexTask.Execute.throw_std_exception", 1, NULL, 0);
build_index_task.to_index_engine_ =
EngineFactory::Build(file->dimension_, file->location_, (EngineType)file->engine_type_,
(MetricType)file->metric_type_, json);
build_index_task.Execute();
fiu_disable("XBuildIndexTask.Execute.throw_std_exception");
fiu_enable("XBuildIndexTask.Execute.update_table_file_fail", 1, NULL, 0);
build_index_task.to_index_engine_ =
EngineFactory::Build(file->dimension_, file->location_, (EngineType)file->engine_type_,
(MetricType)file->metric_type_, json);
build_index_task.Execute();
fiu_disable("XBuildIndexTask.Execute.update_table_file_fail");
fiu_disable("XBuildIndexTask.Execute.throw_std_exception");
fiu_disable("XBuildIndexTask.Execute.has_collection");
fiu_disable("XBuildIndexTask.Execute.create_table_success");
build_index_task.Execute();
// search task
engine::VectorsData vector;
auto search_job = std::make_shared<SearchJob>(dummy_context, 1, 1, vector);
file->metric_type_ = static_cast<int>(MetricType::IP);
file->engine_type_ = static_cast<int>(engine::EngineType::FAISS_IVFSQ8H);
opentracing::mocktracer::MockTracerOptions tracer_options;
auto mock_tracer =
std::shared_ptr<opentracing::Tracer>{new opentracing::mocktracer::MockTracer{std::move(tracer_options)}};
auto mock_span = mock_tracer->StartSpan("mock_span");
auto trace_context = std::make_shared<milvus::tracing::TraceContext>(mock_span);
dummy_context->SetTraceContext(trace_context);
XSearchTask search_task(dummy_context, file, label);
search_task.job_ = search_job;
std::string cpu_resouce_name = "cpu_name1";
std::vector<std::string> path = {cpu_resouce_name};
search_task.task_path_ = Path(path, 0);
ResMgrInst::GetInstance()->Add(std::make_shared<CpuResource>(cpu_resouce_name, 1, true));
search_task.Load(LoadType::CPU2GPU, 0);
search_task.Load(LoadType::GPU2CPU, 0);
fiu_enable("XSearchTask.Load.throw_std_exception", 1, NULL, 0);
search_task.Load(LoadType::GPU2CPU, 0);
fiu_disable("XSearchTask.Load.throw_std_exception");
fiu_enable("XSearchTask.Load.out_of_memory", 1, NULL, 0);
search_task.Load(LoadType::GPU2CPU, 0);
fiu_disable("XSearchTask.Load.out_of_memory");
fiu_enable("XSearchTask.Execute.search_fail", 1, NULL, 0);
search_task.Execute();
fiu_disable("XSearchTask.Execute.search_fail");
fiu_enable("XSearchTask.Execute.throw_std_exception", 1, NULL, 0);
search_task.Execute();
fiu_disable("XSearchTask.Execute.throw_std_exception");
search_task.Execute();
scheduler::ResultIds ids, tar_ids;
scheduler::ResultDistances distances, tar_distances;
XSearchTask::MergeTopkToResultSet(ids, distances, 1, 1, 1, true, tar_ids, tar_distances);
}
//TEST(TaskTest, TEST_TASK) {
// auto dummy_context = std::make_shared<milvus::server::Context>("dummy_request_id");
//
// auto file = std::make_shared<SegmentSchema>();
// file->index_params_ = "{ \"nlist\": 16384 }";
// file->dimension_ = 64;
// auto label = std::make_shared<BroadcastLabel>();
//
// TestTask task(dummy_context, file, label);
// task.Load(LoadType::CPU2GPU, 0);
// auto th = std::thread([&]() {
// task.Execute();
// });
// task.Wait();
//
// if (th.joinable()) {
// th.join();
// }
//
// static const char* CONFIG_PATH = "/tmp/milvus_test";
// auto options = milvus::engine::DBFactory::BuildOption();
// options.meta_.path_ = CONFIG_PATH;
// options.meta_.backend_uri_ = "sqlite://:@:/";
// options.insert_cache_immediately_ = true;
// auto meta_ptr = std::make_shared<milvus::engine::meta::SqliteMetaImpl>(options.meta_);
//
// file->collection_id_ = "111";
// file->location_ = "/tmp/milvus_test/index_file1.txt";
// auto build_index_job = std::make_shared<BuildIndexJob>(meta_ptr, options);
// XBuildIndexTask build_index_task(file, label);
// build_index_task.job_ = build_index_job;
//
// build_index_task.Load(LoadType::TEST, 0);
//
// fiu_init(0);
// fiu_enable("XBuildIndexTask.Load.throw_std_exception", 1, NULL, 0);
// build_index_task.Load(LoadType::TEST, 0);
// fiu_disable("XBuildIndexTask.Load.throw_std_exception");
//
// fiu_enable("XBuildIndexTask.Load.out_of_memory", 1, NULL, 0);
// build_index_task.Load(LoadType::TEST, 0);
// fiu_disable("XBuildIndexTask.Load.out_of_memory");
//
// build_index_task.Execute();
// // always enable 'create_table_success'
// fiu_enable("XBuildIndexTask.Execute.create_table_success", 1, NULL, 0);
//
// milvus::json json = {{"nlist", 16384}};
// build_index_task.to_index_engine_ =
// EngineFactory::Build(file->dimension_, file->location_, (EngineType)file->engine_type_,
// (MetricType)file->metric_type_, json);
//
// build_index_task.Execute();
//
// fiu_enable("XBuildIndexTask.Execute.build_index_fail", 1, NULL, 0);
// build_index_task.to_index_engine_ =
// EngineFactory::Build(file->dimension_, file->location_, (EngineType)file->engine_type_,
// (MetricType)file->metric_type_, json);
// build_index_task.Execute();
// fiu_disable("XBuildIndexTask.Execute.build_index_fail");
//
// // always enable 'has_collection'
// fiu_enable("XBuildIndexTask.Execute.has_collection", 1, NULL, 0);
// build_index_task.to_index_engine_ =
// EngineFactory::Build(file->dimension_, file->location_, (EngineType)file->engine_type_,
// (MetricType)file->metric_type_, json);
// build_index_task.Execute();
//
// fiu_enable("XBuildIndexTask.Execute.throw_std_exception", 1, NULL, 0);
// build_index_task.to_index_engine_ =
// EngineFactory::Build(file->dimension_, file->location_, (EngineType)file->engine_type_,
// (MetricType)file->metric_type_, json);
// build_index_task.Execute();
// fiu_disable("XBuildIndexTask.Execute.throw_std_exception");
//
// fiu_enable("XBuildIndexTask.Execute.update_table_file_fail", 1, NULL, 0);
// build_index_task.to_index_engine_ =
// EngineFactory::Build(file->dimension_, file->location_, (EngineType)file->engine_type_,
// (MetricType)file->metric_type_, json);
// build_index_task.Execute();
// fiu_disable("XBuildIndexTask.Execute.update_table_file_fail");
//
// fiu_disable("XBuildIndexTask.Execute.throw_std_exception");
// fiu_disable("XBuildIndexTask.Execute.has_collection");
// fiu_disable("XBuildIndexTask.Execute.create_table_success");
// build_index_task.Execute();
//
// // search task
// engine::VectorsData vector;
// auto search_job = std::make_shared<SearchJob>(dummy_context, 1, 1, vector);
// file->metric_type_ = static_cast<int>(MetricType::IP);
// file->engine_type_ = static_cast<int>(engine::EngineType::FAISS_IVFSQ8H);
// opentracing::mocktracer::MockTracerOptions tracer_options;
// auto mock_tracer =
// std::shared_ptr<opentracing::Tracer>{new opentracing::mocktracer::MockTracer{std::move(tracer_options)}};
// auto mock_span = mock_tracer->StartSpan("mock_span");
// auto trace_context = std::make_shared<milvus::tracing::TraceContext>(mock_span);
// dummy_context->SetTraceContext(trace_context);
// XSearchTask search_task(dummy_context, file, label);
// search_task.job_ = search_job;
// std::string cpu_resouce_name = "cpu_name1";
// std::vector<std::string> path = {cpu_resouce_name};
// search_task.task_path_ = Path(path, 0);
// ResMgrInst::GetInstance()->Add(std::make_shared<CpuResource>(cpu_resouce_name, 1, true));
//
// search_task.Load(LoadType::CPU2GPU, 0);
// search_task.Load(LoadType::GPU2CPU, 0);
//
// fiu_enable("XSearchTask.Load.throw_std_exception", 1, NULL, 0);
// search_task.Load(LoadType::GPU2CPU, 0);
// fiu_disable("XSearchTask.Load.throw_std_exception");
//
// fiu_enable("XSearchTask.Load.out_of_memory", 1, NULL, 0);
// search_task.Load(LoadType::GPU2CPU, 0);
// fiu_disable("XSearchTask.Load.out_of_memory");
//
// fiu_enable("XSearchTask.Execute.search_fail", 1, NULL, 0);
// search_task.Execute();
// fiu_disable("XSearchTask.Execute.search_fail");
//
// fiu_enable("XSearchTask.Execute.throw_std_exception", 1, NULL, 0);
// search_task.Execute();
// fiu_disable("XSearchTask.Execute.throw_std_exception");
//
// search_task.Execute();
//
// scheduler::ResultIds ids, tar_ids;
// scheduler::ResultDistances distances, tar_distances;
// XSearchTask::MergeTopkToResultSet(ids, distances, 1, 1, 1, true, tar_ids, tar_distances);
//}
TEST(TaskTest, TEST_PATH) {
Path path;
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或