Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
59f45763
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
59f45763
编写于
10月 11, 2019
作者:
Y
Yu Kun
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
MS-636 Add optimizer in scheduler for FAISS_IVFSQ8H
Former-commit-id: 4dff6236c98a86965e686b4a94e5492e0c1d5722
上级
c884b9fd
变更
17
隐藏空白更改
内联
并排
Showing
17 changed file
with
193 addition
and
37 deletion
+193
-37
cpp/CHANGELOG.md
cpp/CHANGELOG.md
+1
-0
cpp/src/db/DBImpl.cpp
cpp/src/db/DBImpl.cpp
+13
-12
cpp/src/scheduler/Algorithm.cpp
cpp/src/scheduler/Algorithm.cpp
+0
-2
cpp/src/scheduler/JobMgr.cpp
cpp/src/scheduler/JobMgr.cpp
+5
-2
cpp/src/scheduler/SchedInst.cpp
cpp/src/scheduler/SchedInst.cpp
+3
-0
cpp/src/scheduler/SchedInst.h
cpp/src/scheduler/SchedInst.h
+24
-0
cpp/src/scheduler/TaskCreator.cpp
cpp/src/scheduler/TaskCreator.cpp
+2
-2
cpp/src/scheduler/action/PushTaskToNeighbour.cpp
cpp/src/scheduler/action/PushTaskToNeighbour.cpp
+27
-16
cpp/src/scheduler/optimizer/HybridPass.cpp
cpp/src/scheduler/optimizer/HybridPass.cpp
+6
-1
cpp/src/scheduler/optimizer/Optimizer.h
cpp/src/scheduler/optimizer/Optimizer.h
+5
-1
cpp/src/scheduler/task/BuildIndexTask.cpp
cpp/src/scheduler/task/BuildIndexTask.cpp
+5
-0
cpp/unittest/CMakeLists.txt
cpp/unittest/CMakeLists.txt
+2
-0
cpp/unittest/db/CMakeLists.txt
cpp/unittest/db/CMakeLists.txt
+9
-0
cpp/unittest/db/appendix/log_config.conf
cpp/unittest/db/appendix/log_config.conf
+27
-0
cpp/unittest/db/appendix/server_config.yaml
cpp/unittest/db/appendix/server_config.yaml
+37
-0
cpp/unittest/db/test_db.cpp
cpp/unittest/db/test_db.cpp
+26
-0
cpp/unittest/db/utils.cpp
cpp/unittest/db/utils.cpp
+1
-1
未找到文件。
cpp/CHANGELOG.md
浏览文件 @
59f45763
...
@@ -37,6 +37,7 @@ Please mark all change in change log and use the ticket from JIRA.
...
@@ -37,6 +37,7 @@ Please mark all change in change log and use the ticket from JIRA.
## New Feature
## New Feature
-
MS-627 - Integrate new index: IVFSQHybrid
-
MS-627 - Integrate new index: IVFSQHybrid
-
MS-631 - IVFSQ8H Index support
-
MS-631 - IVFSQ8H Index support
-
MS-636 - Add optimizer in scheduler for FAISS_IVFSQ8H
## Task
## Task
-
MS-554 - Change license to Apache 2.0
-
MS-554 - Change license to Apache 2.0
...
...
cpp/src/db/DBImpl.cpp
浏览文件 @
59f45763
...
@@ -900,20 +900,21 @@ DBImpl::BackgroundBuildIndex() {
...
@@ -900,20 +900,21 @@ DBImpl::BackgroundBuildIndex() {
meta_ptr_
->
FilesToIndex
(
to_index_files
);
meta_ptr_
->
FilesToIndex
(
to_index_files
);
Status
status
;
Status
status
;
scheduler
::
BuildIndexJobPtr
job
=
std
::
make_shared
<
scheduler
::
BuildIndexJob
>
(
0
,
meta_ptr_
,
options_
);
if
(
!
to_index_files
.
empty
())
{
scheduler
::
BuildIndexJobPtr
job
=
std
::
make_shared
<
scheduler
::
BuildIndexJob
>
(
0
,
meta_ptr_
,
options_
);
// step 2: put build index task to scheduler
// step 2: put build index task to scheduler
for
(
auto
&
file
:
to_index_files
)
{
for
(
auto
&
file
:
to_index_files
)
{
scheduler
::
TableFileSchemaPtr
file_ptr
=
std
::
make_shared
<
meta
::
TableFileSchema
>
(
file
);
scheduler
::
TableFileSchemaPtr
file_ptr
=
std
::
make_shared
<
meta
::
TableFileSchema
>
(
file
);
job
->
AddToIndexFiles
(
file_ptr
);
job
->
AddToIndexFiles
(
file_ptr
);
}
}
scheduler
::
JobMgrInst
::
GetInstance
()
->
Put
(
job
);
scheduler
::
JobMgrInst
::
GetInstance
()
->
Put
(
job
);
job
->
WaitBuildIndexFinish
();
job
->
WaitBuildIndexFinish
();
if
(
!
job
->
GetStatus
().
ok
())
{
if
(
!
job
->
GetStatus
().
ok
())
{
Status
status
=
job
->
GetStatus
();
Status
status
=
job
->
GetStatus
();
ENGINE_LOG_ERROR
<<
"Building index failed: "
<<
status
.
ToString
();
ENGINE_LOG_ERROR
<<
"Building index failed: "
<<
status
.
ToString
();
}
}
}
// for (auto &file : to_index_files) {
// for (auto &file : to_index_files) {
// status = BuildIndex(file);
// status = BuildIndex(file);
// if (!status.ok()) {
// if (!status.ok()) {
...
...
cpp/src/scheduler/Algorithm.cpp
浏览文件 @
59f45763
...
@@ -29,8 +29,6 @@ constexpr uint64_t MAXINT = std::numeric_limits<uint32_t>::max();
...
@@ -29,8 +29,6 @@ constexpr uint64_t MAXINT = std::numeric_limits<uint32_t>::max();
uint64_t
uint64_t
ShortestPath
(
const
ResourcePtr
&
src
,
const
ResourcePtr
&
dest
,
const
ResourceMgrPtr
&
res_mgr
,
ShortestPath
(
const
ResourcePtr
&
src
,
const
ResourcePtr
&
dest
,
const
ResourceMgrPtr
&
res_mgr
,
std
::
vector
<
std
::
string
>&
path
)
{
std
::
vector
<
std
::
string
>&
path
)
{
std
::
vector
<
std
::
vector
<
std
::
string
>>
paths
;
uint64_t
num_of_resources
=
res_mgr
->
GetAllResources
().
size
();
uint64_t
num_of_resources
=
res_mgr
->
GetAllResources
().
size
();
std
::
unordered_map
<
uint64_t
,
std
::
string
>
id_name_map
;
std
::
unordered_map
<
uint64_t
,
std
::
string
>
id_name_map
;
std
::
unordered_map
<
std
::
string
,
uint64_t
>
name_id_map
;
std
::
unordered_map
<
std
::
string
,
uint64_t
>
name_id_map
;
...
...
cpp/src/scheduler/JobMgr.cpp
浏览文件 @
59f45763
...
@@ -16,7 +16,9 @@
...
@@ -16,7 +16,9 @@
// under the License.
// under the License.
#include "scheduler/JobMgr.h"
#include "scheduler/JobMgr.h"
#include "SchedInst.h"
#include "TaskCreator.h"
#include "TaskCreator.h"
#include "optimizer/Optimizer.h"
#include "task/Task.h"
#include "task/Task.h"
#include <src/scheduler/optimizer/Optimizer.h>
#include <src/scheduler/optimizer/Optimizer.h>
...
@@ -67,8 +69,9 @@ JobMgr::worker_function() {
...
@@ -67,8 +69,9 @@ JobMgr::worker_function() {
}
}
auto
tasks
=
build_task
(
job
);
auto
tasks
=
build_task
(
job
);
for
(
auto
&
task
:
tasks
)
{
// TODO: optimizer all task
OptimizerInst
::
GetInstance
()
->
Run
(
task
);
}
// disk resources NEVER be empty.
// disk resources NEVER be empty.
if
(
auto
disk
=
res_mgr_
->
GetDiskResources
()[
0
].
lock
())
{
if
(
auto
disk
=
res_mgr_
->
GetDiskResources
()[
0
].
lock
())
{
...
...
cpp/src/scheduler/SchedInst.cpp
浏览文件 @
59f45763
...
@@ -38,6 +38,9 @@ std::mutex SchedInst::mutex_;
...
@@ -38,6 +38,9 @@ std::mutex SchedInst::mutex_;
scheduler
::
JobMgrPtr
JobMgrInst
::
instance
=
nullptr
;
scheduler
::
JobMgrPtr
JobMgrInst
::
instance
=
nullptr
;
std
::
mutex
JobMgrInst
::
mutex_
;
std
::
mutex
JobMgrInst
::
mutex_
;
OptimizerPtr
OptimizerInst
::
instance
=
nullptr
;
std
::
mutex
OptimizerInst
::
mutex_
;
void
void
load_simple_config
()
{
load_simple_config
()
{
server
::
Config
&
config
=
server
::
Config
::
GetInstance
();
server
::
Config
&
config
=
server
::
Config
::
GetInstance
();
...
...
cpp/src/scheduler/SchedInst.h
浏览文件 @
59f45763
...
@@ -20,9 +20,12 @@
...
@@ -20,9 +20,12 @@
#include "JobMgr.h"
#include "JobMgr.h"
#include "ResourceMgr.h"
#include "ResourceMgr.h"
#include "Scheduler.h"
#include "Scheduler.h"
#include "optimizer/HybridPass.h"
#include "optimizer/Optimizer.h"
#include <memory>
#include <memory>
#include <mutex>
#include <mutex>
#include <vector>
namespace
milvus
{
namespace
milvus
{
namespace
scheduler
{
namespace
scheduler
{
...
@@ -81,6 +84,27 @@ class JobMgrInst {
...
@@ -81,6 +84,27 @@ class JobMgrInst {
static
std
::
mutex
mutex_
;
static
std
::
mutex
mutex_
;
};
};
class
OptimizerInst
{
public:
static
OptimizerPtr
GetInstance
()
{
if
(
instance
==
nullptr
)
{
std
::
lock_guard
<
std
::
mutex
>
lock
(
mutex_
);
if
(
instance
==
nullptr
)
{
HybridPassPtr
pass_ptr
=
std
::
make_shared
<
HybridPass
>
();
std
::
vector
<
PassPtr
>
pass_list
;
pass_list
.
push_back
(
pass_ptr
);
instance
=
std
::
make_shared
<
Optimizer
>
(
pass_list
);
}
}
return
instance
;
}
private:
static
scheduler
::
OptimizerPtr
instance
;
static
std
::
mutex
mutex_
;
};
void
void
StartSchedulerService
();
StartSchedulerService
();
...
...
cpp/src/scheduler/TaskCreator.cpp
浏览文件 @
59f45763
...
@@ -16,10 +16,10 @@
...
@@ -16,10 +16,10 @@
// under the License.
// under the License.
#include "scheduler/TaskCreator.h"
#include "scheduler/TaskCreator.h"
#include <src/scheduler/tasklabel/SpecResLabel.h>
#include "SchedInst.h"
#include "SchedInst.h"
#include "
scheduler/
tasklabel/BroadcastLabel.h"
#include "tasklabel/BroadcastLabel.h"
#include "tasklabel/DefaultLabel.h"
#include "tasklabel/DefaultLabel.h"
#include "tasklabel/SpecResLabel.h"
namespace
milvus
{
namespace
milvus
{
namespace
scheduler
{
namespace
scheduler
{
...
...
cpp/src/scheduler/action/PushTaskToNeighbour.cpp
浏览文件 @
59f45763
...
@@ -19,6 +19,7 @@
...
@@ -19,6 +19,7 @@
#include <random>
#include <random>
#include "../Algorithm.h"
#include "../Algorithm.h"
#include "Action.h"
#include "Action.h"
#include "scheduler/tasklabel/SpecResLabel.h"
#include "src/cache/GpuCacheMgr.h"
#include "src/cache/GpuCacheMgr.h"
#include "src/server/Config.h"
#include "src/server/Config.h"
...
@@ -145,25 +146,35 @@ Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, ResourcePtr
...
@@ -145,25 +146,35 @@ Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, ResourcePtr
paths
.
emplace_back
(
path
);
paths
.
emplace_back
(
path
);
}
}
if
(
task
->
job_
.
lock
()
->
type
()
==
JobType
::
SEARCH
)
{
if
(
task
->
job_
.
lock
()
->
type
()
==
JobType
::
SEARCH
)
{
// step 2: select min cost, cost(resource) = avg_cost * task_to_do + transport_cost
auto
label
=
task
->
label
();
uint64_t
min_cost
=
std
::
numeric_limits
<
uint64_t
>::
max
();
auto
spec_label
=
std
::
static_pointer_cast
<
SpecResLabel
>
(
label
);
uint64_t
min_cost_idx
=
0
;
if
(
spec_label
->
resource
().
lock
()
->
type
()
==
ResourceType
::
CPU
)
{
for
(
uint64_t
i
=
0
;
i
<
compute_resources
.
size
();
++
i
)
{
std
::
vector
<
std
::
string
>
spec_path
;
if
(
compute_resources
[
i
]
->
TotalTasks
()
==
0
)
{
spec_path
.
push_back
(
spec_label
->
resource
().
lock
()
->
name
());
min_cost_idx
=
i
;
spec_path
.
push_back
(
resource
->
name
());
break
;
task
->
path
()
=
Path
(
spec_path
,
spec_path
.
size
()
-
1
);
}
}
else
{
uint64_t
cost
=
// step 2: select min cost, cost(resource) = avg_cost * task_to_do + transport_cost
compute_resources
[
i
]
->
TaskAvgCost
()
*
compute_resources
[
i
]
->
NumOfTaskToExec
()
+
transport_costs
[
i
];
uint64_t
min_cost
=
std
::
numeric_limits
<
uint64_t
>::
max
();
if
(
min_cost
>
cost
)
{
uint64_t
min_cost_idx
=
0
;
min_cost
=
cost
;
for
(
uint64_t
i
=
0
;
i
<
compute_resources
.
size
();
++
i
)
{
min_cost_idx
=
i
;
if
(
compute_resources
[
i
]
->
TotalTasks
()
==
0
)
{
min_cost_idx
=
i
;
break
;
}
uint64_t
cost
=
compute_resources
[
i
]
->
TaskAvgCost
()
*
compute_resources
[
i
]
->
NumOfTaskToExec
()
+
transport_costs
[
i
];
if
(
min_cost
>
cost
)
{
min_cost
=
cost
;
min_cost_idx
=
i
;
}
}
}
// step 3: set path in task
Path
task_path
(
paths
[
min_cost_idx
],
paths
[
min_cost_idx
].
size
()
-
1
);
task
->
path
()
=
task_path
;
}
}
// step 3: set path in task
Path
task_path
(
paths
[
min_cost_idx
],
paths
[
min_cost_idx
].
size
()
-
1
);
task
->
path
()
=
task_path
;
}
else
if
(
task
->
job_
.
lock
()
->
type
()
==
JobType
::
BUILD
)
{
}
else
if
(
task
->
job_
.
lock
()
->
type
()
==
JobType
::
BUILD
)
{
// step2: Read device id in config
// step2: Read device id in config
// get build index gpu resource
// get build index gpu resource
...
...
cpp/src/scheduler/optimizer/HybridPass.cpp
浏览文件 @
59f45763
...
@@ -16,7 +16,9 @@
...
@@ -16,7 +16,9 @@
// under the License.
// under the License.
#include "scheduler/optimizer/HybridPass.h"
#include "scheduler/optimizer/HybridPass.h"
#include "scheduler/SchedInst.h"
#include "scheduler/task/SearchTask.h"
#include "scheduler/task/SearchTask.h"
#include "scheduler/tasklabel/SpecResLabel.h"
namespace
milvus
{
namespace
milvus
{
namespace
scheduler
{
namespace
scheduler
{
...
@@ -28,7 +30,10 @@ HybridPass::Run(const TaskPtr& task) {
...
@@ -28,7 +30,10 @@ HybridPass::Run(const TaskPtr& task) {
return
false
;
return
false
;
auto
search_task
=
std
::
static_pointer_cast
<
XSearchTask
>
(
task
);
auto
search_task
=
std
::
static_pointer_cast
<
XSearchTask
>
(
task
);
if
(
search_task
->
file_
->
engine_type_
==
(
int
)
engine
::
EngineType
::
FAISS_IVFSQ8H
)
{
if
(
search_task
->
file_
->
engine_type_
==
(
int
)
engine
::
EngineType
::
FAISS_IVFSQ8H
)
{
// TODO: make specified label
// TODO: remove "cpu" hardcode
ResourcePtr
res_ptr
=
ResMgrInst
::
GetInstance
()
->
GetResource
(
"cpu"
);
auto
label
=
std
::
make_shared
<
SpecResLabel
>
(
std
::
weak_ptr
<
Resource
>
(
res_ptr
));
task
->
label
()
=
label
;
return
true
;
return
true
;
}
}
return
false
;
return
false
;
...
...
cpp/src/scheduler/optimizer/Optimizer.h
浏览文件 @
59f45763
...
@@ -25,6 +25,7 @@
...
@@ -25,6 +25,7 @@
#include <string>
#include <string>
#include <thread>
#include <thread>
#include <unordered_map>
#include <unordered_map>
#include <utility>
#include <vector>
#include <vector>
#include "Pass.h"
#include "Pass.h"
...
@@ -34,7 +35,8 @@ namespace scheduler {
...
@@ -34,7 +35,8 @@ namespace scheduler {
class
Optimizer
{
class
Optimizer
{
public:
public:
Optimizer
()
=
default
;
explicit
Optimizer
(
std
::
vector
<
PassPtr
>
pass_list
)
:
pass_list_
(
std
::
move
(
pass_list
))
{
}
void
void
Init
();
Init
();
...
@@ -46,5 +48,7 @@ class Optimizer {
...
@@ -46,5 +48,7 @@ class Optimizer {
std
::
vector
<
PassPtr
>
pass_list_
;
std
::
vector
<
PassPtr
>
pass_list_
;
};
};
using
OptimizerPtr
=
std
::
shared_ptr
<
Optimizer
>
;
}
// namespace scheduler
}
// namespace scheduler
}
// namespace milvus
}
// namespace milvus
cpp/src/scheduler/task/BuildIndexTask.cpp
浏览文件 @
59f45763
...
@@ -124,6 +124,7 @@ XBuildIndexTask::Execute() {
...
@@ -124,6 +124,7 @@ XBuildIndexTask::Execute() {
ENGINE_LOG_ERROR
<<
"Failed to create table file: "
<<
status
.
ToString
();
ENGINE_LOG_ERROR
<<
"Failed to create table file: "
<<
status
.
ToString
();
build_index_job
->
BuildIndexDone
(
to_index_id_
);
build_index_job
->
BuildIndexDone
(
to_index_id_
);
build_index_job
->
GetStatus
()
=
status
;
build_index_job
->
GetStatus
()
=
status
;
to_index_engine_
=
nullptr
;
return
;
return
;
}
}
...
@@ -136,6 +137,7 @@ XBuildIndexTask::Execute() {
...
@@ -136,6 +137,7 @@ XBuildIndexTask::Execute() {
ENGINE_LOG_DEBUG
<<
"Failed to update file to index, mark file: "
<<
table_file
.
file_id_
ENGINE_LOG_DEBUG
<<
"Failed to update file to index, mark file: "
<<
table_file
.
file_id_
<<
" to to_delete"
;
<<
" to to_delete"
;
to_index_engine_
=
nullptr
;
return
;
return
;
}
}
}
catch
(
std
::
exception
&
ex
)
{
}
catch
(
std
::
exception
&
ex
)
{
...
@@ -150,6 +152,7 @@ XBuildIndexTask::Execute() {
...
@@ -150,6 +152,7 @@ XBuildIndexTask::Execute() {
<<
std
::
endl
;
<<
std
::
endl
;
build_index_job
->
GetStatus
()
=
Status
(
DB_ERROR
,
msg
);
build_index_job
->
GetStatus
()
=
Status
(
DB_ERROR
,
msg
);
to_index_engine_
=
nullptr
;
return
;
return
;
}
}
...
@@ -158,6 +161,7 @@ XBuildIndexTask::Execute() {
...
@@ -158,6 +161,7 @@ XBuildIndexTask::Execute() {
meta_ptr
->
HasTable
(
file_
->
table_id_
,
has_table
);
meta_ptr
->
HasTable
(
file_
->
table_id_
,
has_table
);
if
(
!
has_table
)
{
if
(
!
has_table
)
{
meta_ptr
->
DeleteTableFiles
(
file_
->
table_id_
);
meta_ptr
->
DeleteTableFiles
(
file_
->
table_id_
);
to_index_engine_
=
nullptr
;
return
;
return
;
}
}
...
@@ -177,6 +181,7 @@ XBuildIndexTask::Execute() {
...
@@ -177,6 +181,7 @@ XBuildIndexTask::Execute() {
<<
", possible out of disk space"
<<
std
::
endl
;
<<
", possible out of disk space"
<<
std
::
endl
;
build_index_job
->
GetStatus
()
=
Status
(
DB_ERROR
,
msg
);
build_index_job
->
GetStatus
()
=
Status
(
DB_ERROR
,
msg
);
to_index_engine_
=
nullptr
;
return
;
return
;
}
}
...
...
cpp/unittest/CMakeLists.txt
浏览文件 @
59f45763
...
@@ -50,6 +50,7 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_files)
...
@@ -50,6 +50,7 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_files)
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/scheduler/job scheduler_job_files
)
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/scheduler/job scheduler_job_files
)
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/scheduler/resource scheduler_resource_files
)
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/scheduler/resource scheduler_resource_files
)
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/scheduler/task scheduler_task_files
)
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/scheduler/task scheduler_task_files
)
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/scheduler/optimizer scheduler_optimizer_files
)
set
(
scheduler_files
set
(
scheduler_files
${
scheduler_main_files
}
${
scheduler_main_files
}
${
scheduler_action_files
}
${
scheduler_action_files
}
...
@@ -57,6 +58,7 @@ set(scheduler_files
...
@@ -57,6 +58,7 @@ set(scheduler_files
${
scheduler_job_files
}
${
scheduler_job_files
}
${
scheduler_resource_files
}
${
scheduler_resource_files
}
${
scheduler_task_files
}
${
scheduler_task_files
}
${
scheduler_optimizer_files
}
)
)
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/server server_files
)
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/server server_files
)
...
...
cpp/unittest/db/CMakeLists.txt
浏览文件 @
59f45763
...
@@ -31,3 +31,12 @@ target_link_libraries(test_db
...
@@ -31,3 +31,12 @@ target_link_libraries(test_db
install
(
TARGETS test_db DESTINATION unittest
)
install
(
TARGETS test_db DESTINATION unittest
)
configure_file
(
appendix/server_config.yaml
"
${
CMAKE_CURRENT_BINARY_DIR
}
/milvus/conf/server_config.yaml"
COPYONLY
)
configure_file
(
appendix/log_config.conf
"
${
CMAKE_CURRENT_BINARY_DIR
}
/milvus/conf/log_config.conf"
COPYONLY
)
cpp/unittest/db/appendix/log_config.conf
0 → 100644
浏览文件 @
59f45763
*
GLOBAL
:
FORMAT
=
"%datetime | %level | %logger | %msg"
FILENAME
=
"/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-global.log"
ENABLED
=
true
TO_FILE
=
true
TO_STANDARD_OUTPUT
=
false
SUBSECOND_PRECISION
=
3
PERFORMANCE_TRACKING
=
false
MAX_LOG_FILE_SIZE
=
209715200
## Throw log files away after 200MB
*
DEBUG
:
FILENAME
=
"/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-debug.log"
ENABLED
=
true
*
WARNING
:
FILENAME
=
"/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-warning.log"
*
TRACE
:
FILENAME
=
"/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-trace.log"
*
VERBOSE
:
FORMAT
=
"%datetime{%d/%M/%y} | %level-%vlevel | %msg"
TO_FILE
=
false
TO_STANDARD_OUTPUT
=
false
## Error logs
*
ERROR
:
ENABLED
=
true
FILENAME
=
"/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-error.log"
*
FATAL
:
ENABLED
=
true
FILENAME
=
"/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-fatal.log"
cpp/unittest/db/appendix/server_config.yaml
0 → 100644
浏览文件 @
59f45763
# All the following configurations are default values.
server_config
:
address
:
0.0.0.0
# milvus server ip address (IPv4)
port
:
19530
# port range: 1025 ~ 65534
deploy_mode
:
single
# deployment type: single, cluster_readonly, cluster_writable
time_zone
:
UTC+8
db_config
:
primary_path
:
/tmp/milvus
# path used to store data and meta
secondary_path
:
# path used to store data only, split by semicolon
backend_url
:
sqlite://:@:/
# URI format: dialect://username:password@host:port/database
# Keep 'dialect://:@:/', and replace other texts with real values.
# Replace 'dialect' with 'mysql' or 'sqlite'
insert_buffer_size
:
4
# GB, maximum insert buffer size allowed
build_index_gpu
:
0
# gpu id used for building index
metric_config
:
enable_monitor
:
false
# enable monitoring or not
collector
:
prometheus
# prometheus
prometheus_config
:
port
:
8080
# port prometheus used to fetch metrics
cache_config
:
cpu_mem_capacity
:
16
# GB, CPU memory used for cache
cpu_mem_threshold
:
0.85
# percentage of data kept when cache cleanup triggered
cache_insert_data
:
false
# whether load inserted data into cache
engine_config
:
blas_threshold
:
20
resource_config
:
resource_pool
:
-
cpu
-
gpu0
cpp/unittest/db/test_db.cpp
浏览文件 @
59f45763
...
@@ -23,14 +23,18 @@
...
@@ -23,14 +23,18 @@
#include "db/DBFactory.h"
#include "db/DBFactory.h"
#include "cache/CpuCacheMgr.h"
#include "cache/CpuCacheMgr.h"
#include "utils/CommonUtil.h"
#include "utils/CommonUtil.h"
#include "server/Config.h"
#include <gtest/gtest.h>
#include <gtest/gtest.h>
#include <boost/filesystem.hpp>
#include <boost/filesystem.hpp>
#include <thread>
#include <thread>
#include <random>
#include <random>
namespace
{
namespace
{
static
const
char
*
CONFIG_FILE_PATH
=
"./milvus/conf/server_config.yaml"
;
static
const
char
*
TABLE_NAME
=
"test_group"
;
static
const
char
*
TABLE_NAME
=
"test_group"
;
static
constexpr
int64_t
TABLE_DIM
=
256
;
static
constexpr
int64_t
TABLE_DIM
=
256
;
static
constexpr
int64_t
VECTOR_COUNT
=
25000
;
static
constexpr
int64_t
VECTOR_COUNT
=
25000
;
...
@@ -228,6 +232,9 @@ TEST_F(DBTest, DB_TEST) {
...
@@ -228,6 +232,9 @@ TEST_F(DBTest, DB_TEST) {
}
}
TEST_F
(
DBTest
,
SEARCH_TEST
)
{
TEST_F
(
DBTest
,
SEARCH_TEST
)
{
milvus
::
server
::
Config
&
config
=
milvus
::
server
::
Config
::
GetInstance
();
milvus
::
Status
s
=
config
.
LoadConfigFile
(
CONFIG_FILE_PATH
);
milvus
::
engine
::
meta
::
TableSchema
table_info
=
BuildTableSchema
();
milvus
::
engine
::
meta
::
TableSchema
table_info
=
BuildTableSchema
();
auto
stat
=
db_
->
CreateTable
(
table_info
);
auto
stat
=
db_
->
CreateTable
(
table_info
);
...
@@ -290,6 +297,25 @@ TEST_F(DBTest, SEARCH_TEST) {
...
@@ -290,6 +297,25 @@ TEST_F(DBTest, SEARCH_TEST) {
ASSERT_TRUE
(
stat
.
ok
());
ASSERT_TRUE
(
stat
.
ok
());
}
}
//test FAISS_IVFSQ8H optimizer
index
.
engine_type_
=
(
int
)
milvus
::
engine
::
EngineType
::
FAISS_IVFSQ8H
;
db_
->
CreateIndex
(
TABLE_NAME
,
index
);
// wait until build index finish
{
milvus
::
engine
::
QueryResults
results
;
stat
=
db_
->
Query
(
TABLE_NAME
,
k
,
nq
,
10
,
xq
.
data
(),
results
);
ASSERT_TRUE
(
stat
.
ok
());
}
{
//search by specify index file
milvus
::
engine
::
meta
::
DatesT
dates
;
std
::
vector
<
std
::
string
>
file_ids
=
{
"1"
,
"2"
,
"3"
,
"4"
,
"5"
,
"6"
};
milvus
::
engine
::
QueryResults
results
;
stat
=
db_
->
Query
(
TABLE_NAME
,
file_ids
,
k
,
nq
,
10
,
xq
.
data
(),
dates
,
results
);
ASSERT_TRUE
(
stat
.
ok
());
}
// TODO(lxj): add groundTruth assert
// TODO(lxj): add groundTruth assert
}
}
...
...
cpp/unittest/db/utils.cpp
浏览文件 @
59f45763
...
@@ -97,7 +97,7 @@ DBTest::SetUp() {
...
@@ -97,7 +97,7 @@ DBTest::SetUp() {
auto
res_mgr
=
milvus
::
scheduler
::
ResMgrInst
::
GetInstance
();
auto
res_mgr
=
milvus
::
scheduler
::
ResMgrInst
::
GetInstance
();
res_mgr
->
Clear
();
res_mgr
->
Clear
();
res_mgr
->
Add
(
milvus
::
scheduler
::
ResourceFactory
::
Create
(
"disk"
,
"DISK"
,
0
,
true
,
false
));
res_mgr
->
Add
(
milvus
::
scheduler
::
ResourceFactory
::
Create
(
"disk"
,
"DISK"
,
0
,
true
,
false
));
res_mgr
->
Add
(
milvus
::
scheduler
::
ResourceFactory
::
Create
(
"cpu"
,
"CPU"
,
0
,
true
,
fals
e
));
res_mgr
->
Add
(
milvus
::
scheduler
::
ResourceFactory
::
Create
(
"cpu"
,
"CPU"
,
0
,
true
,
tru
e
));
res_mgr
->
Add
(
milvus
::
scheduler
::
ResourceFactory
::
Create
(
"gtx1660"
,
"GPU"
,
0
,
true
,
true
));
res_mgr
->
Add
(
milvus
::
scheduler
::
ResourceFactory
::
Create
(
"gtx1660"
,
"GPU"
,
0
,
true
,
true
));
auto
default_conn
=
milvus
::
scheduler
::
Connection
(
"IO"
,
500.0
);
auto
default_conn
=
milvus
::
scheduler
::
Connection
(
"IO"
,
500.0
);
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录