Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
e51b7f16
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
e51b7f16
编写于
7月 24, 2019
作者:
S
starlord
浏览文件
操作
浏览文件
下载
差异文件
Merge remote-tracking branch 'source/branch-0.3.1' into branch-0.3.1
Former-commit-id: 6093fec932087c5c92d82e454d76bd614090615e
上级
ed029178
afc40c45
变更
16
显示空白变更内容
内联
并排
Showing
16 changed file
with
229 addition
and
137 deletion
+229
-137
cpp/src/cache/Cache.cpp
cpp/src/cache/Cache.cpp
+1
-1
cpp/src/cache/CacheMgr.cpp
cpp/src/cache/CacheMgr.cpp
+13
-0
cpp/src/cache/CpuCacheMgr.cpp
cpp/src/cache/CpuCacheMgr.cpp
+5
-1
cpp/src/cache/GpuCacheMgr.cpp
cpp/src/cache/GpuCacheMgr.cpp
+5
-1
cpp/src/config/ConfigNode.cpp
cpp/src/config/ConfigNode.cpp
+1
-1
cpp/src/db/Constants.h
cpp/src/db/Constants.h
+6
-6
cpp/src/db/DB.cpp
cpp/src/db/DB.cpp
+1
-2
cpp/src/db/DB.h
cpp/src/db/DB.h
+1
-1
cpp/src/db/DBMetaImpl.h
cpp/src/db/DBMetaImpl.h
+56
-35
cpp/src/db/IDGenerator.cpp
cpp/src/db/IDGenerator.cpp
+3
-1
cpp/src/db/IDGenerator.h
cpp/src/db/IDGenerator.h
+21
-10
cpp/src/db/Meta.cpp
cpp/src/db/Meta.cpp
+2
-0
cpp/src/db/Meta.h
cpp/src/db/Meta.h
+62
-32
cpp/src/db/MySQLMetaImpl.h
cpp/src/db/MySQLMetaImpl.h
+44
-43
cpp/src/db/scheduler/TaskDispatchStrategy.cpp
cpp/src/db/scheduler/TaskDispatchStrategy.cpp
+3
-0
cpp/src/db/scheduler/TaskScheduler.cpp
cpp/src/db/scheduler/TaskScheduler.cpp
+5
-3
未找到文件。
cpp/src/cache/Cache.cpp
浏览文件 @
e51b7f16
...
...
@@ -89,7 +89,7 @@ void Cache::erase(const std::string& key) {
const
DataObjPtr
&
data_ptr
=
obj_ptr
->
data_
;
usage_
-=
data_ptr
->
size
();
SERVER_LOG_DEBUG
<<
"Erase "
<<
key
<<
"
from cache"
;
SERVER_LOG_DEBUG
<<
"Erase "
<<
key
<<
"
size: "
<<
data_ptr
->
size
()
;
lru_
.
erase
(
key
);
}
...
...
cpp/src/cache/CacheMgr.cpp
浏览文件 @
e51b7f16
...
...
@@ -4,6 +4,7 @@
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include "utils/Log.h"
#include "CacheMgr.h"
#include "metrics/Metrics.h"
...
...
@@ -20,6 +21,7 @@ CacheMgr::~CacheMgr() {
uint64_t
CacheMgr
::
ItemCount
()
const
{
if
(
cache_
==
nullptr
)
{
SERVER_LOG_ERROR
<<
"Cache doesn't exist"
;
return
0
;
}
...
...
@@ -28,6 +30,7 @@ uint64_t CacheMgr::ItemCount() const {
bool
CacheMgr
::
ItemExists
(
const
std
::
string
&
key
)
{
if
(
cache_
==
nullptr
)
{
SERVER_LOG_ERROR
<<
"Cache doesn't exist"
;
return
false
;
}
...
...
@@ -36,6 +39,7 @@ bool CacheMgr::ItemExists(const std::string& key) {
DataObjPtr
CacheMgr
::
GetItem
(
const
std
::
string
&
key
)
{
if
(
cache_
==
nullptr
)
{
SERVER_LOG_ERROR
<<
"Cache doesn't exist"
;
return
nullptr
;
}
server
::
Metrics
::
GetInstance
().
CacheAccessTotalIncrement
();
...
...
@@ -45,6 +49,7 @@ DataObjPtr CacheMgr::GetItem(const std::string& key) {
engine
::
Index_ptr
CacheMgr
::
GetIndex
(
const
std
::
string
&
key
)
{
DataObjPtr
obj
=
GetItem
(
key
);
if
(
obj
!=
nullptr
)
{
SERVER_LOG_ERROR
<<
"Can't get object from key: "
<<
key
;
return
obj
->
data
();
}
...
...
@@ -53,6 +58,7 @@ engine::Index_ptr CacheMgr::GetIndex(const std::string& key) {
void
CacheMgr
::
InsertItem
(
const
std
::
string
&
key
,
const
DataObjPtr
&
data
)
{
if
(
cache_
==
nullptr
)
{
SERVER_LOG_ERROR
<<
"Cache doesn't exist"
;
return
;
}
...
...
@@ -62,6 +68,7 @@ void CacheMgr::InsertItem(const std::string& key, const DataObjPtr& data) {
void
CacheMgr
::
InsertItem
(
const
std
::
string
&
key
,
const
engine
::
Index_ptr
&
index
)
{
if
(
cache_
==
nullptr
)
{
SERVER_LOG_ERROR
<<
"Cache doesn't exist"
;
return
;
}
...
...
@@ -72,6 +79,7 @@ void CacheMgr::InsertItem(const std::string& key, const engine::Index_ptr& index
void
CacheMgr
::
EraseItem
(
const
std
::
string
&
key
)
{
if
(
cache_
==
nullptr
)
{
SERVER_LOG_ERROR
<<
"Cache doesn't exist"
;
return
;
}
...
...
@@ -81,6 +89,7 @@ void CacheMgr::EraseItem(const std::string& key) {
void
CacheMgr
::
PrintInfo
()
{
if
(
cache_
==
nullptr
)
{
SERVER_LOG_ERROR
<<
"Cache doesn't exist"
;
return
;
}
...
...
@@ -89,6 +98,7 @@ void CacheMgr::PrintInfo() {
void
CacheMgr
::
ClearCache
()
{
if
(
cache_
==
nullptr
)
{
SERVER_LOG_ERROR
<<
"Cache doesn't exist"
;
return
;
}
...
...
@@ -97,6 +107,7 @@ void CacheMgr::ClearCache() {
int64_t
CacheMgr
::
CacheUsage
()
const
{
if
(
cache_
==
nullptr
)
{
SERVER_LOG_ERROR
<<
"Cache doesn't exist"
;
return
0
;
}
...
...
@@ -105,6 +116,7 @@ int64_t CacheMgr::CacheUsage() const {
int64_t
CacheMgr
::
CacheCapacity
()
const
{
if
(
cache_
==
nullptr
)
{
SERVER_LOG_ERROR
<<
"Cache doesn't exist"
;
return
0
;
}
...
...
@@ -113,6 +125,7 @@ int64_t CacheMgr::CacheCapacity() const {
void
CacheMgr
::
SetCapacity
(
int64_t
capacity
)
{
if
(
cache_
==
nullptr
)
{
SERVER_LOG_ERROR
<<
"Cache doesn't exist"
;
return
;
}
cache_
->
set_capacity
(
capacity
);
...
...
cpp/src/cache/CpuCacheMgr.cpp
浏览文件 @
e51b7f16
...
...
@@ -12,10 +12,14 @@ namespace zilliz {
namespace
milvus
{
namespace
cache
{
namespace
{
constexpr
int64_t
unit
=
1024
*
1024
*
1024
;
}
CpuCacheMgr
::
CpuCacheMgr
()
{
server
::
ConfigNode
&
config
=
server
::
ServerConfig
::
GetInstance
().
GetConfig
(
server
::
CONFIG_CACHE
);
int64_t
cap
=
config
.
GetInt64Value
(
server
::
CONFIG_CPU_CACHE_CAPACITY
,
16
);
cap
*=
1024
*
1024
*
1024
;
cap
*=
unit
;
cache_
=
std
::
make_shared
<
Cache
>
(
cap
,
1UL
<<
32
);
double
free_percent
=
config
.
GetDoubleValue
(
server
::
CACHE_FREE_PERCENT
,
0.85
);
...
...
cpp/src/cache/GpuCacheMgr.cpp
浏览文件 @
e51b7f16
...
...
@@ -11,10 +11,14 @@ namespace zilliz {
namespace
milvus
{
namespace
cache
{
namespace
{
constexpr
int64_t
unit
=
1024
*
1024
*
1024
;
}
GpuCacheMgr
::
GpuCacheMgr
()
{
server
::
ConfigNode
&
config
=
server
::
ServerConfig
::
GetInstance
().
GetConfig
(
server
::
CONFIG_CACHE
);
int64_t
cap
=
config
.
GetInt64Value
(
server
::
CONFIG_GPU_CACHE_CAPACITY
,
1
);
cap
*=
1024
*
1024
*
1024
;
cap
*=
unit
;
cache_
=
std
::
make_shared
<
Cache
>
(
cap
,
1UL
<<
32
);
}
...
...
cpp/src/config/ConfigNode.cpp
浏览文件 @
e51b7f16
...
...
@@ -94,7 +94,7 @@ double
ConfigNode
::
GetDoubleValue
(
const
std
::
string
&
param_key
,
double
default_val
)
const
{
std
::
string
val
=
GetValue
(
param_key
);
if
(
!
val
.
empty
())
{
return
std
::
strto
l
d
(
val
.
c_str
(),
nullptr
);
return
std
::
strtod
(
val
.
c_str
(),
nullptr
);
}
else
{
return
default_val
;
}
...
...
cpp/src/db/Constants.h
浏览文件 @
e51b7f16
...
...
@@ -9,14 +9,14 @@ namespace zilliz {
namespace
milvus
{
namespace
engine
{
const
size_t
K
=
1024UL
;
const
size_t
M
=
K
*
K
;
const
size_t
G
=
K
*
M
;
const
size_t
T
=
K
*
G
;
const
expr
size_t
K
=
1024UL
;
const
expr
size_t
M
=
K
*
K
;
const
expr
size_t
G
=
K
*
M
;
const
expr
size_t
T
=
K
*
G
;
const
size_t
MAX_TABLE_FILE_MEM
=
128
*
M
;
const
expr
size_t
MAX_TABLE_FILE_MEM
=
128
*
M
;
const
int
VECTOR_TYPE_SIZE
=
sizeof
(
float
);
const
expr
int
VECTOR_TYPE_SIZE
=
sizeof
(
float
);
}
// namespace engine
}
// namespace milvus
...
...
cpp/src/db/DB.cpp
浏览文件 @
e51b7f16
...
...
@@ -12,11 +12,10 @@ namespace zilliz {
namespace
milvus
{
namespace
engine
{
DB
::~
DB
()
{}
DB
::~
DB
()
=
default
;
void
DB
::
Open
(
const
Options
&
options
,
DB
**
dbptr
)
{
*
dbptr
=
DBFactory
::
Build
(
options
);
return
;
}
}
// namespace engine
...
...
cpp/src/db/DB.h
浏览文件 @
e51b7f16
...
...
@@ -52,7 +52,7 @@ public:
DB
(
const
DB
&
)
=
delete
;
DB
&
operator
=
(
const
DB
&
)
=
delete
;
virtual
~
DB
();
virtual
~
DB
()
=
0
;
};
// DB
}
// namespace engine
...
...
cpp/src/db/DBMetaImpl.h
浏览文件 @
e51b7f16
...
...
@@ -8,67 +8,88 @@
#include "Meta.h"
#include "Options.h"
namespace
zilliz
{
namespace
milvus
{
namespace
engine
{
namespace
meta
{
auto
StoragePrototype
(
const
std
::
string
&
path
);
auto
StoragePrototype
(
const
std
::
string
&
path
);
class
DBMetaImpl
:
public
Meta
{
public:
DBMetaImpl
(
const
DBMetaOptions
&
options_
);
public:
explicit
DBMetaImpl
(
const
DBMetaOptions
&
options_
);
Status
CreateTable
(
TableSchema
&
table_schema
)
override
;
Status
DescribeTable
(
TableSchema
&
group_info_
)
override
;
Status
HasTable
(
const
std
::
string
&
table_id
,
bool
&
has_or_not
)
override
;
Status
AllTables
(
std
::
vector
<
TableSchema
>
&
table_schema_array
)
override
;
Status
DeleteTable
(
const
std
::
string
&
table_id
)
override
;
virtual
Status
CreateTable
(
TableSchema
&
table_schema
)
override
;
virtual
Status
DescribeTable
(
TableSchema
&
group_info_
)
override
;
virtual
Status
HasTable
(
const
std
::
string
&
table_id
,
bool
&
has_or_not
)
override
;
virtual
Status
AllTables
(
std
::
vector
<
TableSchema
>&
table_schema_array
)
override
;
Status
DeleteTableFiles
(
const
std
::
string
&
table_id
)
override
;
virtual
Status
DeleteTable
(
const
std
::
string
&
table_id
)
override
;
virtual
Status
DeleteTableFiles
(
const
std
::
string
&
table_id
)
override
;
Status
CreateTableFile
(
TableFileSchema
&
file_schema
)
override
;
virtual
Status
CreateTableFile
(
TableFileSchema
&
file_schema
)
override
;
virtual
Status
DropPartitionsByDates
(
const
std
::
string
&
table_id
,
const
DatesT
&
dates
)
override
;
Status
DropPartitionsByDates
(
const
std
::
string
&
table_id
,
const
DatesT
&
dates
)
override
;
virtual
Status
GetTableFiles
(
const
std
::
string
&
table_id
,
const
std
::
vector
<
size_t
>&
ids
,
TableFilesSchema
&
table_files
)
override
;
Status
GetTableFiles
(
const
std
::
string
&
table_id
,
const
std
::
vector
<
size_t
>
&
ids
,
TableFilesSchema
&
table_files
)
override
;
virtual
Status
HasNonIndexFiles
(
const
std
::
string
&
table_id
,
bool
&
has
)
override
;
Status
HasNonIndexFiles
(
const
std
::
string
&
table_id
,
bool
&
has
)
override
;
virtual
Status
UpdateTableFilesToIndex
(
const
std
::
string
&
table_id
)
override
;
Status
UpdateTableFilesToIndex
(
const
std
::
string
&
table_id
)
override
;
virtual
Status
UpdateTableFile
(
TableFileSchema
&
file_schema
)
override
;
Status
UpdateTableFile
(
TableFileSchema
&
file_schema
)
override
;
virtual
Status
UpdateTableFiles
(
TableFilesSchema
&
files
)
override
;
Status
UpdateTableFiles
(
TableFilesSchema
&
files
)
override
;
virtual
Status
FilesToSearch
(
const
std
::
string
&
table_id
,
const
DatesT
&
partition
,
DatePartionedTableFilesSchema
&
files
)
override
;
Status
FilesToSearch
(
const
std
::
string
&
table_id
,
const
DatesT
&
partition
,
DatePartionedTableFilesSchema
&
files
)
override
;
virtual
Status
FilesToMerge
(
const
std
::
string
&
table_id
,
DatePartionedTableFilesSchema
&
files
)
override
;
Status
FilesToMerge
(
const
std
::
string
&
table_id
,
DatePartionedTableFilesSchema
&
files
)
override
;
virtual
Status
FilesToIndex
(
TableFilesSchema
&
)
override
;
Status
FilesToIndex
(
TableFilesSchema
&
)
override
;
virtual
Status
Archive
()
override
;
Status
Archive
()
override
;
virtual
Status
Size
(
uint64_t
&
result
)
override
;
Status
Size
(
uint64_t
&
result
)
override
;
virtual
Status
CleanUp
()
override
;
Status
CleanUp
()
override
;
virtual
Status
CleanUpFilesWithTTL
(
uint16_t
seconds
)
override
;
Status
CleanUpFilesWithTTL
(
uint16_t
seconds
)
override
;
virtual
Status
DropAll
()
override
;
Status
DropAll
()
override
;
virtual
Status
Count
(
const
std
::
string
&
table_id
,
uint64_t
&
result
)
override
;
Status
Count
(
const
std
::
string
&
table_id
,
uint64_t
&
result
)
override
;
virtual
~
DBMetaImpl
()
;
~
DBMetaImpl
()
override
;
private:
Status
NextFileId
(
std
::
string
&
file_id
);
Status
NextTableId
(
std
::
string
&
table_id
);
private:
Status
NextFileId
(
std
::
string
&
file_id
);
Status
NextTableId
(
std
::
string
&
table_id
);
Status
DiscardFiles
(
long
to_discard_size
);
Status
Initialize
();
...
...
cpp/src/db/IDGenerator.cpp
浏览文件 @
e51b7f16
...
...
@@ -13,7 +13,9 @@ namespace zilliz {
namespace
milvus
{
namespace
engine
{
IDGenerator
::~
IDGenerator
()
{}
IDGenerator
::~
IDGenerator
()
=
default
;
constexpr
size_t
SimpleIDGenerator
::
MAX_IDS_PER_MICRO
;
IDNumber
SimpleIDGenerator
::
GetNextIDNumber
()
{
auto
now
=
std
::
chrono
::
system_clock
::
now
();
...
...
cpp/src/db/IDGenerator.h
浏览文件 @
e51b7f16
...
...
@@ -10,28 +10,39 @@
#include <cstddef>
#include <vector>
namespace
zilliz
{
namespace
milvus
{
namespace
engine
{
class
IDGenerator
{
public:
virtual
IDNumber
GetNextIDNumber
()
=
0
;
virtual
void
GetNextIDNumbers
(
size_t
n
,
IDNumbers
&
ids
)
=
0
;
public:
virtual
IDNumber
GetNextIDNumber
(
)
=
0
;
virtual
~
IDGenerator
();
virtual
void
GetNextIDNumbers
(
size_t
n
,
IDNumbers
&
ids
)
=
0
;
virtual
~
IDGenerator
()
=
0
;
};
// IDGenerator
class
SimpleIDGenerator
:
public
IDGenerator
{
public:
virtual
IDNumber
GetNextIDNumber
()
override
;
virtual
void
GetNextIDNumbers
(
size_t
n
,
IDNumbers
&
ids
)
override
;
public:
~
SimpleIDGenerator
()
override
=
default
;
IDNumber
GetNextIDNumber
()
override
;
void
GetNextIDNumbers
(
size_t
n
,
IDNumbers
&
ids
)
override
;
private:
void
NextIDNumbers
(
size_t
n
,
IDNumbers
&
ids
);
private:
void
NextIDNumbers
(
size_t
n
,
IDNumbers
&
ids
);
const
size_t
MAX_IDS_PER_MICRO
=
1000
;
static
constexpr
size_t
MAX_IDS_PER_MICRO
=
1000
;
};
// SimpleIDGenerator
...
...
cpp/src/db/Meta.cpp
浏览文件 @
e51b7f16
...
...
@@ -13,6 +13,8 @@ namespace milvus {
namespace
engine
{
namespace
meta
{
Meta
::~
Meta
()
=
default
;
DateT
Meta
::
GetDate
(
const
std
::
time_t
&
t
,
int
day_delta
)
{
struct
tm
ltm
;
localtime_r
(
&
t
,
&
ltm
);
...
...
cpp/src/db/Meta.h
浏览文件 @
e51b7f16
...
...
@@ -20,56 +20,86 @@ namespace meta {
class
Meta
{
public:
public:
using
Ptr
=
std
::
shared_ptr
<
Meta
>
;
virtual
Status
CreateTable
(
TableSchema
&
table_schema
)
=
0
;
virtual
Status
DescribeTable
(
TableSchema
&
table_schema
)
=
0
;
virtual
Status
HasTable
(
const
std
::
string
&
table_id
,
bool
&
has_or_not
)
=
0
;
virtual
Status
AllTables
(
std
::
vector
<
TableSchema
>&
table_schema_array
)
=
0
;
virtual
~
Meta
()
=
0
;
virtual
Status
DeleteTable
(
const
std
::
string
&
table_id
)
=
0
;
virtual
Status
DeleteTableFiles
(
const
std
::
string
&
table_id
)
=
0
;
virtual
Status
CreateTable
(
TableSchema
&
table_schema
)
=
0
;
virtual
Status
CreateTableFile
(
TableFileSchema
&
file_schema
)
=
0
;
virtual
Status
DropPartitionsByDates
(
const
std
::
string
&
table_id
,
const
DatesT
&
dates
)
=
0
;
virtual
Status
DescribeTable
(
TableSchema
&
table_schema
)
=
0
;
virtual
Status
GetTableFiles
(
const
std
::
string
&
table_id
,
const
std
::
vector
<
size_t
>&
ids
,
TableFilesSchema
&
table_files
)
=
0
;
virtual
Status
HasTable
(
const
std
::
string
&
table_id
,
bool
&
has_or_not
)
=
0
;
virtual
Status
UpdateTableFilesToIndex
(
const
std
::
string
&
table_id
)
=
0
;
virtual
Status
AllTables
(
std
::
vector
<
TableSchema
>
&
table_schema_array
)
=
0
;
virtual
Status
UpdateTableFile
(
TableFileSchema
&
file_schema
)
=
0
;
virtual
Status
DeleteTable
(
const
std
::
string
&
table_id
)
=
0
;
virtual
Status
UpdateTableFiles
(
TableFilesSchema
&
files
)
=
0
;
virtual
Status
DeleteTableFiles
(
const
std
::
string
&
table_id
)
=
0
;
virtual
Status
FilesToSearch
(
const
std
::
string
&
table_id
,
const
DatesT
&
partition
,
DatePartionedTableFilesSchema
&
files
)
=
0
;
virtual
Status
CreateTableFile
(
TableFileSchema
&
file_schema
)
=
0
;
virtual
Status
FilesToMerge
(
const
std
::
string
&
table_id
,
DatePartionedTableFilesSchema
&
fil
es
)
=
0
;
virtual
Status
DropPartitionsByDates
(
const
std
::
string
&
table_id
,
const
DatesT
&
dat
es
)
=
0
;
virtual
Status
Size
(
uint64_t
&
result
)
=
0
;
virtual
Status
GetTableFiles
(
const
std
::
string
&
table_id
,
const
std
::
vector
<
size_t
>
&
ids
,
TableFilesSchema
&
table_files
)
=
0
;
virtual
Status
Archive
()
=
0
;
virtual
Status
UpdateTableFilesToIndex
(
const
std
::
string
&
table_id
)
=
0
;
virtual
Status
FilesToIndex
(
TableFilesSchema
&
)
=
0
;
virtual
Status
UpdateTableFile
(
TableFileSchema
&
file_schema
)
=
0
;
virtual
Status
HasNonIndexFiles
(
const
std
::
string
&
table_id
,
bool
&
has
)
=
0
;
virtual
Status
UpdateTableFiles
(
TableFilesSchema
&
files
)
=
0
;
virtual
Status
CleanUp
()
=
0
;
virtual
Status
CleanUpFilesWithTTL
(
uint16_t
)
=
0
;
virtual
Status
FilesToSearch
(
const
std
::
string
&
table_id
,
const
DatesT
&
partition
,
DatePartionedTableFilesSchema
&
files
)
=
0
;
virtual
Status
DropAll
()
=
0
;
virtual
Status
FilesToMerge
(
const
std
::
string
&
table_id
,
DatePartionedTableFilesSchema
&
files
)
=
0
;
virtual
Status
Count
(
const
std
::
string
&
table_id
,
uint64_t
&
result
)
=
0
;
virtual
Status
Size
(
uint64_t
&
result
)
=
0
;
static
DateT
GetDate
(
const
std
::
time_t
&
t
,
int
day_delta
=
0
);
static
DateT
GetDate
();
static
DateT
GetDateWithDelta
(
int
day_delta
);
virtual
Status
Archive
()
=
0
;
virtual
Status
FilesToIndex
(
TableFilesSchema
&
)
=
0
;
virtual
Status
HasNonIndexFiles
(
const
std
::
string
&
table_id
,
bool
&
has
)
=
0
;
virtual
Status
CleanUp
()
=
0
;
virtual
Status
CleanUpFilesWithTTL
(
uint16_t
)
=
0
;
virtual
Status
DropAll
()
=
0
;
virtual
Status
Count
(
const
std
::
string
&
table_id
,
uint64_t
&
result
)
=
0
;
static
DateT
GetDate
(
const
std
::
time_t
&
t
,
int
day_delta
=
0
);
static
DateT
GetDate
();
static
DateT
GetDateWithDelta
(
int
day_delta
);
};
// MetaData
...
...
cpp/src/db/MySQLMetaImpl.h
浏览文件 @
e51b7f16
...
...
@@ -12,68 +12,69 @@
#include "mysql++/mysql++.h"
#include <mutex>
namespace
zilliz
{
namespace
milvus
{
namespace
engine
{
namespace
meta
{
// auto StoragePrototype(const std::string& path);
using
namespace
mysqlpp
;
using
namespace
mysqlpp
;
class
MySQLMetaImpl
:
public
Meta
{
class
MySQLMetaImpl
:
public
Meta
{
public:
MySQLMetaImpl
(
const
DBMetaOptions
&
options_
,
const
int
&
mode
);
MySQLMetaImpl
(
const
DBMetaOptions
&
options_
,
const
int
&
mode
);
virtual
Status
CreateTable
(
TableSchema
&
table_schema
)
override
;
virtual
Status
DescribeTable
(
TableSchema
&
group_info_
)
override
;
virtual
Status
HasTable
(
const
std
::
string
&
table_id
,
bool
&
has_or_not
)
override
;
virtual
Status
AllTables
(
std
::
vector
<
TableSchema
>&
table_schema_array
)
override
;
Status
CreateTable
(
TableSchema
&
table_schema
)
override
;
Status
DescribeTable
(
TableSchema
&
group_info_
)
override
;
Status
HasTable
(
const
std
::
string
&
table_id
,
bool
&
has_or_not
)
override
;
Status
AllTables
(
std
::
vector
<
TableSchema
>
&
table_schema_array
)
override
;
virtual
Status
DeleteTable
(
const
std
::
string
&
table_id
)
override
;
virtual
Status
DeleteTableFiles
(
const
std
::
string
&
table_id
)
override
;
Status
DeleteTable
(
const
std
::
string
&
table_id
)
override
;
Status
DeleteTableFiles
(
const
std
::
string
&
table_id
)
override
;
virtual
Status
CreateTableFile
(
TableFileSchema
&
file_schema
)
override
;
virtual
Status
DropPartitionsByDates
(
const
std
::
string
&
table_id
,
const
DatesT
&
dates
)
override
;
Status
CreateTableFile
(
TableFileSchema
&
file_schema
)
override
;
Status
DropPartitionsByDates
(
const
std
::
string
&
table_id
,
const
DatesT
&
dates
)
override
;
virtual
Status
GetTableFiles
(
const
std
::
string
&
table_id
,
const
std
::
vector
<
size_t
>&
ids
,
TableFilesSchema
&
table_files
)
override
;
Status
GetTableFiles
(
const
std
::
string
&
table_id
,
const
std
::
vector
<
size_t
>
&
ids
,
TableFilesSchema
&
table_files
)
override
;
virtual
Status
HasNonIndexFiles
(
const
std
::
string
&
table_id
,
bool
&
has
)
override
;
Status
HasNonIndexFiles
(
const
std
::
string
&
table_id
,
bool
&
has
)
override
;
virtual
Status
UpdateTableFile
(
TableFileSchema
&
file_schema
)
override
;
Status
UpdateTableFile
(
TableFileSchema
&
file_schema
)
override
;
virtual
Status
UpdateTableFilesToIndex
(
const
std
::
string
&
table_id
)
override
;
Status
UpdateTableFilesToIndex
(
const
std
::
string
&
table_id
)
override
;
virtual
Status
UpdateTableFiles
(
TableFilesSchema
&
files
)
override
;
Status
UpdateTableFiles
(
TableFilesSchema
&
files
)
override
;
virtual
Status
FilesToSearch
(
const
std
::
string
&
table_id
,
const
DatesT
&
partition
,
DatePartionedTableFilesSchema
&
files
)
override
;
Status
FilesToSearch
(
const
std
::
string
&
table_id
,
const
DatesT
&
partition
,
DatePartionedTableFilesSchema
&
files
)
override
;
virtual
Status
FilesToMerge
(
const
std
::
string
&
table_id
,
DatePartionedTableFilesSchema
&
files
)
override
;
Status
FilesToMerge
(
const
std
::
string
&
table_id
,
DatePartionedTableFilesSchema
&
files
)
override
;
virtual
Status
FilesToIndex
(
TableFilesSchema
&
)
override
;
Status
FilesToIndex
(
TableFilesSchema
&
)
override
;
virtual
Status
Archive
()
override
;
Status
Archive
()
override
;
virtual
Status
Size
(
uint64_t
&
result
)
override
;
Status
Size
(
uint64_t
&
result
)
override
;
virtual
Status
CleanUp
()
override
;
Status
CleanUp
()
override
;
virtual
Status
CleanUpFilesWithTTL
(
uint16_t
seconds
)
override
;
Status
CleanUpFilesWithTTL
(
uint16_t
seconds
)
override
;
virtual
Status
DropAll
()
override
;
Status
DropAll
()
override
;
virtual
Status
Count
(
const
std
::
string
&
table_id
,
uint64_t
&
result
)
override
;
Status
Count
(
const
std
::
string
&
table_id
,
uint64_t
&
result
)
override
;
virtual
~
MySQLMetaImpl
();
private:
Status
NextFileId
(
std
::
string
&
file_id
);
Status
NextTableId
(
std
::
string
&
table_id
);
Status
NextFileId
(
std
::
string
&
file_id
);
Status
NextTableId
(
std
::
string
&
table_id
);
Status
DiscardFiles
(
long
long
to_discard_size
);
Status
Initialize
();
...
...
@@ -84,7 +85,7 @@ namespace meta {
bool
safe_grab
=
false
;
// std::mutex connectionMutex_;
};
// DBMetaImpl
};
// DBMetaImpl
}
// namespace meta
}
// namespace engine
...
...
cpp/src/db/scheduler/TaskDispatchStrategy.cpp
浏览文件 @
e51b7f16
...
...
@@ -20,6 +20,7 @@ class ReuseCacheIndexStrategy {
public:
bool
Schedule
(
const
SearchContextPtr
&
context
,
std
::
list
<
ScheduleTaskPtr
>&
task_list
)
{
if
(
context
==
nullptr
)
{
ENGINE_LOG_ERROR
<<
"Task Dispatch context doesn't exist"
;
return
false
;
}
...
...
@@ -64,6 +65,7 @@ class DeleteTableStrategy {
public:
bool
Schedule
(
const
DeleteContextPtr
&
context
,
std
::
list
<
ScheduleTaskPtr
>
&
task_list
)
{
if
(
context
==
nullptr
)
{
ENGINE_LOG_ERROR
<<
"Task Dispatch context doesn't exist"
;
return
false
;
}
...
...
@@ -103,6 +105,7 @@ public:
bool
TaskDispatchStrategy
::
Schedule
(
const
ScheduleContextPtr
&
context_ptr
,
std
::
list
<
zilliz
::
milvus
::
engine
::
ScheduleTaskPtr
>
&
task_list
)
{
if
(
context_ptr
==
nullptr
)
{
ENGINE_LOG_ERROR
<<
"Task Dispatch context doesn't exist"
;
return
false
;
}
...
...
cpp/src/db/scheduler/TaskScheduler.cpp
浏览文件 @
e51b7f16
...
...
@@ -31,6 +31,7 @@ TaskScheduler& TaskScheduler::GetInstance() {
bool
TaskScheduler
::
Start
()
{
if
(
!
stopped_
)
{
SERVER_LOG_INFO
<<
"Task Scheduler isn't started"
;
return
true
;
}
...
...
@@ -47,6 +48,7 @@ TaskScheduler::Start() {
bool
TaskScheduler
::
Stop
()
{
if
(
stopped_
)
{
SERVER_LOG_INFO
<<
"Task Scheduler already stopped"
;
return
true
;
}
...
...
@@ -80,7 +82,7 @@ TaskScheduler::TaskDispatchWorker() {
ScheduleTaskPtr
task_ptr
=
task_dispatch_queue_
.
Take
();
if
(
task_ptr
==
nullptr
)
{
SERVER_LOG_INFO
<<
"Stop db task dispatch thread"
;
break
;
//exit
return
true
;
}
//execute task
...
...
@@ -98,8 +100,8 @@ TaskScheduler::TaskWorker() {
while
(
true
)
{
ScheduleTaskPtr
task_ptr
=
task_queue_
.
Take
();
if
(
task_ptr
==
nullptr
)
{
SERVER_LOG_INFO
<<
"Stop db task thread"
;
break
;
//exit
SERVER_LOG_INFO
<<
"Stop db task
worker
thread"
;
return
true
;
}
//execute task
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录