Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
a0e4d19e
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
a0e4d19e
编写于
9月 27, 2019
作者:
S
starlord
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
format db code
Former-commit-id: 3b4781e4c000beea41baddf54635d7bbe7ea5f52
上级
6ed49d1d
变更
40
展开全部
隐藏空白更改
内联
并排
Showing
40 changed file
with
1174 addition
and
1091 deletion
+1174
-1091
cpp/src/db/Constants.h
cpp/src/db/Constants.h
+2
-2
cpp/src/db/DB.h
cpp/src/db/DB.h
+24
-24
cpp/src/db/DBFactory.cpp
cpp/src/db/DBFactory.cpp
+6
-4
cpp/src/db/DBFactory.h
cpp/src/db/DBFactory.h
+5
-6
cpp/src/db/DBImpl.cpp
cpp/src/db/DBImpl.cpp
+167
-127
cpp/src/db/DBImpl.h
cpp/src/db/DBImpl.h
+6
-6
cpp/src/db/IDGenerator.cpp
cpp/src/db/IDGenerator.cpp
+12
-10
cpp/src/db/IDGenerator.h
cpp/src/db/IDGenerator.h
+0
-2
cpp/src/db/Options.cpp
cpp/src/db/Options.cpp
+13
-10
cpp/src/db/Options.h
cpp/src/db/Options.h
+15
-10
cpp/src/db/Types.h
cpp/src/db/Types.h
+5
-4
cpp/src/db/Utils.cpp
cpp/src/db/Utils.cpp
+44
-30
cpp/src/db/Utils.h
cpp/src/db/Utils.h
+22
-11
cpp/src/db/engine/EngineFactory.cpp
cpp/src/db/engine/EngineFactory.cpp
+10
-9
cpp/src/db/engine/EngineFactory.h
cpp/src/db/engine/EngineFactory.h
+8
-5
cpp/src/db/engine/ExecutionEngine.h
cpp/src/db/engine/ExecutionEngine.h
+11
-12
cpp/src/db/engine/ExecutionEngineImpl.cpp
cpp/src/db/engine/ExecutionEngineImpl.cpp
+51
-36
cpp/src/db/engine/ExecutionEngineImpl.h
cpp/src/db/engine/ExecutionEngineImpl.h
+17
-15
cpp/src/db/insert/MemManager.h
cpp/src/db/insert/MemManager.h
+2
-3
cpp/src/db/insert/MemManagerImpl.cpp
cpp/src/db/insert/MemManagerImpl.cpp
+27
-21
cpp/src/db/insert/MemManagerImpl.h
cpp/src/db/insert/MemManagerImpl.h
+5
-4
cpp/src/db/insert/MemMenagerFactory.cpp
cpp/src/db/insert/MemMenagerFactory.cpp
+4
-2
cpp/src/db/insert/MemMenagerFactory.h
cpp/src/db/insert/MemMenagerFactory.h
+6
-4
cpp/src/db/insert/MemTable.cpp
cpp/src/db/insert/MemTable.cpp
+18
-12
cpp/src/db/insert/MemTable.h
cpp/src/db/insert/MemTable.h
+4
-3
cpp/src/db/insert/MemTableFile.cpp
cpp/src/db/insert/MemTableFile.cpp
+21
-17
cpp/src/db/insert/MemTableFile.h
cpp/src/db/insert/MemTableFile.h
+5
-9
cpp/src/db/insert/VectorSource.cpp
cpp/src/db/insert/VectorSource.cpp
+18
-17
cpp/src/db/insert/VectorSource.h
cpp/src/db/insert/VectorSource.h
+2
-3
cpp/src/db/meta/Meta.h
cpp/src/db/meta/Meta.h
+7
-6
cpp/src/db/meta/MetaConsts.h
cpp/src/db/meta/MetaConsts.h
+10
-10
cpp/src/db/meta/MetaFactory.cpp
cpp/src/db/meta/MetaFactory.cpp
+36
-31
cpp/src/db/meta/MetaFactory.h
cpp/src/db/meta/MetaFactory.h
+6
-5
cpp/src/db/meta/MetaTypes.h
cpp/src/db/meta/MetaTypes.h
+6
-5
cpp/src/db/meta/MySQLConnectionPool.cpp
cpp/src/db/meta/MySQLConnectionPool.cpp
+55
-50
cpp/src/db/meta/MySQLConnectionPool.h
cpp/src/db/meta/MySQLConnectionPool.h
+10
-14
cpp/src/db/meta/MySQLMetaImpl.cpp
cpp/src/db/meta/MySQLMetaImpl.cpp
+290
-300
cpp/src/db/meta/MySQLMetaImpl.h
cpp/src/db/meta/MySQLMetaImpl.h
+10
-11
cpp/src/db/meta/SqliteMetaImpl.cpp
cpp/src/db/meta/SqliteMetaImpl.cpp
+204
-234
cpp/src/db/meta/SqliteMetaImpl.h
cpp/src/db/meta/SqliteMetaImpl.h
+10
-7
未找到文件。
cpp/src/db/Constants.h
浏览文件 @
a0e4d19e
...
...
@@ -33,8 +33,8 @@ constexpr uint64_t MAX_TABLE_FILE_MEM = 128 * M;
constexpr
int
VECTOR_TYPE_SIZE
=
sizeof
(
float
);
static
constexpr
uint64_t
ONE_KB
=
K
;
static
constexpr
uint64_t
ONE_MB
=
ONE_KB
*
ONE_KB
;
static
constexpr
uint64_t
ONE_GB
=
ONE_KB
*
ONE_MB
;
static
constexpr
uint64_t
ONE_MB
=
ONE_KB
*
ONE_KB
;
static
constexpr
uint64_t
ONE_GB
=
ONE_KB
*
ONE_MB
;
}
// namespace engine
}
// namespace milvus
...
...
cpp/src/db/DB.h
浏览文件 @
a0e4d19e
...
...
@@ -25,6 +25,7 @@
#include <string>
#include <memory>
#include <vector>
namespace
zilliz
{
namespace
milvus
{
...
...
@@ -33,46 +34,45 @@ namespace engine {
class
Env
;
class
DB
{
public:
public:
DB
()
=
default
;
DB
(
const
DB
&
)
=
delete
;
DB
&
operator
=
(
const
DB
&
)
=
delete
;
DB
(
const
DB
&
)
=
delete
;
DB
&
operator
=
(
const
DB
&
)
=
delete
;
virtual
~
DB
()
=
default
;
virtual
Status
Start
()
=
0
;
virtual
Status
Stop
()
=
0
;
virtual
Status
CreateTable
(
meta
::
TableSchema
&
table_schema_
)
=
0
;
virtual
Status
DeleteTable
(
const
std
::
string
&
table_id
,
const
meta
::
DatesT
&
dates
)
=
0
;
virtual
Status
DescribeTable
(
meta
::
TableSchema
&
table_schema_
)
=
0
;
virtual
Status
HasTable
(
const
std
::
string
&
table_id
,
bool
&
has_or_not_
)
=
0
;
virtual
Status
AllTables
(
std
::
vector
<
meta
::
TableSchema
>
&
table_schema_array
)
=
0
;
virtual
Status
GetTableRowCount
(
const
std
::
string
&
table_id
,
uint64_t
&
row_count
)
=
0
;
virtual
Status
PreloadTable
(
const
std
::
string
&
table_id
)
=
0
;
virtual
Status
CreateTable
(
meta
::
TableSchema
&
table_schema_
)
=
0
;
virtual
Status
DeleteTable
(
const
std
::
string
&
table_id
,
const
meta
::
DatesT
&
dates
)
=
0
;
virtual
Status
DescribeTable
(
meta
::
TableSchema
&
table_schema_
)
=
0
;
virtual
Status
HasTable
(
const
std
::
string
&
table_id
,
bool
&
has_or_not_
)
=
0
;
virtual
Status
AllTables
(
std
::
vector
<
meta
::
TableSchema
>
&
table_schema_array
)
=
0
;
virtual
Status
GetTableRowCount
(
const
std
::
string
&
table_id
,
uint64_t
&
row_count
)
=
0
;
virtual
Status
PreloadTable
(
const
std
::
string
&
table_id
)
=
0
;
virtual
Status
UpdateTableFlag
(
const
std
::
string
&
table_id
,
int64_t
flag
)
=
0
;
virtual
Status
InsertVectors
(
const
std
::
string
&
table_id_
,
uint64_t
n
,
const
float
*
vectors
,
IDNumbers
&
vector_ids_
)
=
0
;
virtual
Status
InsertVectors
(
const
std
::
string
&
table_id_
,
uint64_t
n
,
const
float
*
vectors
,
IDNumbers
&
vector_ids_
)
=
0
;
virtual
Status
Query
(
const
std
::
string
&
table_id
,
uint64_t
k
,
uint64_t
nq
,
uint64_t
nprobe
,
const
float
*
vectors
,
QueryResults
&
results
)
=
0
;
virtual
Status
Query
(
const
std
::
string
&
table_id
,
uint64_t
k
,
uint64_t
nq
,
uint64_t
nprobe
,
const
float
*
vectors
,
QueryResults
&
results
)
=
0
;
virtual
Status
Query
(
const
std
::
string
&
table_id
,
uint64_t
k
,
uint64_t
nq
,
uint64_t
nprobe
,
const
float
*
vectors
,
const
meta
::
DatesT
&
dates
,
QueryResults
&
results
)
=
0
;
virtual
Status
Query
(
const
std
::
string
&
table_id
,
uint64_t
k
,
uint64_t
nq
,
uint64_t
nprobe
,
const
float
*
vectors
,
const
meta
::
DatesT
&
dates
,
QueryResults
&
results
)
=
0
;
virtual
Status
Query
(
const
std
::
string
&
table_id
,
const
std
::
vector
<
std
::
string
>&
file_ids
,
uint64_t
k
,
uint64_t
nq
,
uint64_t
nprobe
,
const
float
*
vectors
,
const
meta
::
DatesT
&
dates
,
QueryResults
&
results
)
=
0
;
virtual
Status
Query
(
const
std
::
string
&
table_id
,
const
std
::
vector
<
std
::
string
>
&
file_ids
,
uint64_t
k
,
uint64_t
nq
,
uint64_t
nprobe
,
const
float
*
vectors
,
const
meta
::
DatesT
&
dates
,
QueryResults
&
results
)
=
0
;
virtual
Status
Size
(
uint64_t
&
result
)
=
0
;
virtual
Status
Size
(
uint64_t
&
result
)
=
0
;
virtual
Status
CreateIndex
(
const
std
::
string
&
table_id
,
const
TableIndex
&
index
)
=
0
;
virtual
Status
DescribeIndex
(
const
std
::
string
&
table_id
,
TableIndex
&
index
)
=
0
;
virtual
Status
DropIndex
(
const
std
::
string
&
table_id
)
=
0
;
virtual
Status
CreateIndex
(
const
std
::
string
&
table_id
,
const
TableIndex
&
index
)
=
0
;
virtual
Status
DescribeIndex
(
const
std
::
string
&
table_id
,
TableIndex
&
index
)
=
0
;
virtual
Status
DropIndex
(
const
std
::
string
&
table_id
)
=
0
;
virtual
Status
DropAll
()
=
0
;
};
// DB
using
DBPtr
=
std
::
shared_ptr
<
DB
>
;
...
...
cpp/src/db/DBFactory.cpp
浏览文件 @
a0e4d19e
...
...
@@ -16,7 +16,7 @@
// under the License.
#include "DBFactory.h"
#include "
db/
DBFactory.h"
#include "DBImpl.h"
#include "utils/Exception.h"
#include "meta/MetaFactory.h"
...
...
@@ -33,14 +33,16 @@ namespace zilliz {
namespace
milvus
{
namespace
engine
{
DBOptions
DBFactory
::
BuildOption
()
{
DBOptions
DBFactory
::
BuildOption
()
{
auto
meta
=
MetaFactory
::
BuildOption
();
DBOptions
options
;
DBOptions
options
;
options
.
meta_
=
meta
;
return
options
;
}
DBPtr
DBFactory
::
Build
(
const
DBOptions
&
options
)
{
DBPtr
DBFactory
::
Build
(
const
DBOptions
&
options
)
{
return
std
::
make_shared
<
DBImpl
>
(
options
);
}
...
...
cpp/src/db/DBFactory.h
浏览文件 @
a0e4d19e
...
...
@@ -28,13 +28,12 @@ namespace milvus {
namespace
engine
{
class
DBFactory
{
public:
public:
static
DBOptions
BuildOption
();
static
DBPtr
Build
(
const
DBOptions
&
options
);
static
DBPtr
Build
(
const
DBOptions
&
options
);
};
}
}
}
}
// namespace engine
}
// namespace milvus
}
// namespace zilliz
cpp/src/db/DBImpl.cpp
浏览文件 @
a0e4d19e
此差异已折叠。
点击以展开。
cpp/src/db/DBImpl.h
浏览文件 @
a0e4d19e
...
...
@@ -29,7 +29,8 @@
#include <thread>
#include <list>
#include <set>
#include <vector>
#include <string>
namespace
zilliz
{
namespace
milvus
{
...
...
@@ -68,11 +69,11 @@ class DBImpl : public DB {
Status
InsertVectors
(
const
std
::
string
&
table_id
,
uint64_t
n
,
const
float
*
vectors
,
IDNumbers
&
vector_ids
)
override
;
Status
CreateIndex
(
const
std
::
string
&
table_id
,
const
TableIndex
&
index
)
override
;
Status
CreateIndex
(
const
std
::
string
&
table_id
,
const
TableIndex
&
index
)
override
;
Status
DescribeIndex
(
const
std
::
string
&
table_id
,
TableIndex
&
index
)
override
;
Status
DescribeIndex
(
const
std
::
string
&
table_id
,
TableIndex
&
index
)
override
;
Status
DropIndex
(
const
std
::
string
&
table_id
)
override
;
Status
DropIndex
(
const
std
::
string
&
table_id
)
override
;
Status
Query
(
const
std
::
string
&
table_id
,
uint64_t
k
,
...
...
@@ -123,7 +124,7 @@ class DBImpl : public DB {
Status
BackgroundMergeFiles
(
const
std
::
string
&
table_id
);
void
BackgroundCompaction
(
std
::
set
<
std
::
string
>
table_ids
);
void
StartBuildIndexTask
(
bool
force
=
false
);
void
StartBuildIndexTask
(
bool
force
=
false
);
void
BackgroundBuildIndex
();
Status
BuildIndex
(
const
meta
::
TableFileSchema
&
);
...
...
@@ -151,7 +152,6 @@ class DBImpl : public DB {
std
::
list
<
std
::
future
<
void
>>
index_thread_results_
;
std
::
mutex
build_index_mutex_
;
};
// DBImpl
...
...
cpp/src/db/IDGenerator.cpp
浏览文件 @
a0e4d19e
...
...
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#include "IDGenerator.h"
#include "
db/
IDGenerator.h"
#include <chrono>
#include <assert.h>
...
...
@@ -29,16 +29,18 @@ IDGenerator::~IDGenerator() = default;
constexpr
size_t
SimpleIDGenerator
::
MAX_IDS_PER_MICRO
;
IDNumber
SimpleIDGenerator
::
GetNextIDNumber
()
{
IDNumber
SimpleIDGenerator
::
GetNextIDNumber
()
{
auto
now
=
std
::
chrono
::
system_clock
::
now
();
auto
micros
=
std
::
chrono
::
duration_cast
<
std
::
chrono
::
microseconds
>
(
now
.
time_since_epoch
()).
count
();
now
.
time_since_epoch
()).
count
();
return
micros
*
MAX_IDS_PER_MICRO
;
}
void
SimpleIDGenerator
::
NextIDNumbers
(
size_t
n
,
IDNumbers
&
ids
)
{
void
SimpleIDGenerator
::
NextIDNumbers
(
size_t
n
,
IDNumbers
&
ids
)
{
if
(
n
>
MAX_IDS_PER_MICRO
)
{
NextIDNumbers
(
n
-
MAX_IDS_PER_MICRO
,
ids
);
NextIDNumbers
(
n
-
MAX_IDS_PER_MICRO
,
ids
);
NextIDNumbers
(
MAX_IDS_PER_MICRO
,
ids
);
return
;
}
...
...
@@ -48,20 +50,20 @@ void SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers& ids) {
auto
now
=
std
::
chrono
::
system_clock
::
now
();
auto
micros
=
std
::
chrono
::
duration_cast
<
std
::
chrono
::
microseconds
>
(
now
.
time_since_epoch
()).
count
();
now
.
time_since_epoch
()).
count
();
micros
*=
MAX_IDS_PER_MICRO
;
for
(
int
pos
=
0
;
pos
<
n
;
++
pos
)
{
ids
.
push_back
(
micros
+
pos
);
for
(
int
pos
=
0
;
pos
<
n
;
++
pos
)
{
ids
.
push_back
(
micros
+
pos
);
}
}
void
SimpleIDGenerator
::
GetNextIDNumbers
(
size_t
n
,
IDNumbers
&
ids
)
{
void
SimpleIDGenerator
::
GetNextIDNumbers
(
size_t
n
,
IDNumbers
&
ids
)
{
ids
.
clear
();
NextIDNumbers
(
n
,
ids
);
}
}
// namespace engine
}
// namespace milvus
}
// namespace zilliz
cpp/src/db/IDGenerator.h
浏览文件 @
a0e4d19e
...
...
@@ -22,7 +22,6 @@
#include <cstddef>
#include <vector>
namespace
zilliz
{
namespace
milvus
{
namespace
engine
{
...
...
@@ -55,7 +54,6 @@ class SimpleIDGenerator : public IDGenerator {
NextIDNumbers
(
size_t
n
,
IDNumbers
&
ids
);
static
constexpr
size_t
MAX_IDS_PER_MICRO
=
1000
;
};
// SimpleIDGenerator
...
...
cpp/src/db/Options.cpp
浏览文件 @
a0e4d19e
...
...
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#include "Options.h"
#include "
db/
Options.h"
#include "utils/Exception.h"
#include "utils/Log.h"
...
...
@@ -27,18 +27,20 @@ namespace zilliz {
namespace
milvus
{
namespace
engine
{
ArchiveConf
::
ArchiveConf
(
const
std
::
string
&
type
,
const
std
::
string
&
criterias
)
{
ArchiveConf
::
ArchiveConf
(
const
std
::
string
&
type
,
const
std
::
string
&
criterias
)
{
ParseType
(
type
);
ParseCritirias
(
criterias
);
}
void
ArchiveConf
::
SetCriterias
(
const
ArchiveConf
::
CriteriaT
&
criterial
)
{
for
(
auto
&
pair
:
criterial
)
{
void
ArchiveConf
::
SetCriterias
(
const
ArchiveConf
::
CriteriaT
&
criterial
)
{
for
(
auto
&
pair
:
criterial
)
{
criterias_
[
pair
.
first
]
=
pair
.
second
;
}
}
void
ArchiveConf
::
ParseCritirias
(
const
std
::
string
&
criterias
)
{
void
ArchiveConf
::
ParseCritirias
(
const
std
::
string
&
criterias
)
{
std
::
stringstream
ss
(
criterias
);
std
::
vector
<
std
::
string
>
tokens
;
...
...
@@ -48,8 +50,8 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) {
return
;
}
for
(
auto
&
token
:
tokens
)
{
if
(
token
.
empty
())
{
for
(
auto
&
token
:
tokens
)
{
if
(
token
.
empty
())
{
continue
;
}
...
...
@@ -67,12 +69,12 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) {
auto
value
=
std
::
stoi
(
kv
[
1
]);
criterias_
[
kv
[
0
]]
=
value
;
}
catch
(
std
::
out_of_range
&
)
{
catch
(
std
::
out_of_range
&
)
{
std
::
string
msg
=
"Out of range: '"
+
kv
[
1
]
+
"'"
;
ENGINE_LOG_ERROR
<<
msg
;
throw
InvalidArgumentException
(
msg
);
}
catch
(...){
catch
(...)
{
std
::
string
msg
=
"Invalid argument: '"
+
kv
[
1
]
+
"'"
;
ENGINE_LOG_ERROR
<<
msg
;
throw
InvalidArgumentException
(
msg
);
...
...
@@ -80,7 +82,8 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) {
}
}
void
ArchiveConf
::
ParseType
(
const
std
::
string
&
type
)
{
void
ArchiveConf
::
ParseType
(
const
std
::
string
&
type
)
{
if
(
type
!=
"delete"
&&
type
!=
"swap"
)
{
std
::
string
msg
=
"Invalid argument: type='"
+
type
+
"'"
;
throw
InvalidArgumentException
(
msg
);
...
...
cpp/src/db/Options.h
浏览文件 @
a0e4d19e
...
...
@@ -30,22 +30,27 @@ namespace engine {
class
Env
;
static
const
char
*
ARCHIVE_CONF_DISK
=
"disk"
;
static
const
char
*
ARCHIVE_CONF_DAYS
=
"days"
;
static
const
char
*
ARCHIVE_CONF_DISK
=
"disk"
;
static
const
char
*
ARCHIVE_CONF_DAYS
=
"days"
;
struct
ArchiveConf
{
using
CriteriaT
=
std
::
map
<
std
::
string
,
int
>
;
ArchiveConf
(
const
std
::
string
&
type
,
const
std
::
string
&
criterias
=
std
::
string
());
explicit
ArchiveConf
(
const
std
::
string
&
type
,
const
std
::
string
&
criterias
=
std
::
string
());
const
std
::
string
&
GetType
()
const
{
return
type_
;
}
const
CriteriaT
GetCriterias
()
const
{
return
criterias_
;
}
const
std
::
string
&
GetType
()
const
{
return
type_
;
}
void
SetCriterias
(
const
ArchiveConf
::
CriteriaT
&
criterial
);
const
CriteriaT
GetCriterias
()
const
{
return
criterias_
;
}
private:
void
ParseCritirias
(
const
std
::
string
&
type
);
void
ParseType
(
const
std
::
string
&
criterias
);
void
SetCriterias
(
const
ArchiveConf
::
CriteriaT
&
criterial
);
private:
void
ParseCritirias
(
const
std
::
string
&
type
);
void
ParseType
(
const
std
::
string
&
criterias
);
std
::
string
type_
;
CriteriaT
criterias_
;
...
...
@@ -65,7 +70,7 @@ struct DBOptions {
CLUSTER_WRITABLE
}
MODE
;
uint16_t
merge_trigger_number_
=
2
;
uint16_t
merge_trigger_number_
=
2
;
DBMetaOptions
meta_
;
int
mode_
=
MODE
::
SINGLE
;
...
...
cpp/src/db/Types.h
浏览文件 @
a0e4d19e
...
...
@@ -21,22 +21,23 @@
#include <vector>
#include <stdint.h>
#include <utility>
namespace
zilliz
{
namespace
milvus
{
namespace
engine
{
typedef
long
IDNumber
;
typedef
IDNumber
*
IDNumberPtr
;
typedef
int64_t
IDNumber
;
typedef
IDNumber
*
IDNumberPtr
;
typedef
std
::
vector
<
IDNumber
>
IDNumbers
;
typedef
std
::
vector
<
std
::
pair
<
IDNumber
,
double
>>
QueryResult
;
typedef
std
::
vector
<
QueryResult
>
QueryResults
;
struct
TableIndex
{
int32_t
engine_type_
=
(
int
)
EngineType
::
FAISS_IDMAP
;
int32_t
engine_type_
=
(
int
)
EngineType
::
FAISS_IDMAP
;
int32_t
nlist_
=
16384
;
int32_t
metric_type_
=
(
int
)
MetricType
::
L2
;
int32_t
metric_type_
=
(
int
)
MetricType
::
L2
;
};
}
// namespace engine
...
...
cpp/src/db/Utils.cpp
浏览文件 @
a0e4d19e
...
...
@@ -15,13 +15,14 @@
// specific language governing permissions and limitations
// under the License.
#include "Utils.h"
#include "
db/
Utils.h"
#include "utils/CommonUtil.h"
#include "utils/Log.h"
#include <mutex>
#include <chrono>
#include <regex>
#include <vector>
#include <boost/filesystem.hpp>
namespace
zilliz
{
...
...
@@ -31,23 +32,25 @@ namespace utils {
namespace
{
const
char
*
TABLES_FOLDER
=
"/tables/"
;
const
char
*
TABLES_FOLDER
=
"/tables/"
;
uint64_t
index_file_counter
=
0
;
std
::
mutex
index_file_counter_mutex
;
std
::
string
ConstructParentFolder
(
const
std
::
string
&
db_path
,
const
meta
::
TableFileSchema
&
table_file
)
{
std
::
string
ConstructParentFolder
(
const
std
::
string
&
db_path
,
const
meta
::
TableFileSchema
&
table_file
)
{
std
::
string
table_path
=
db_path
+
TABLES_FOLDER
+
table_file
.
table_id_
;
std
::
string
partition_path
=
table_path
+
"/"
+
std
::
to_string
(
table_file
.
date_
);
return
partition_path
;
}
std
::
string
GetTableFileParentFolder
(
const
DBMetaOptions
&
options
,
const
meta
::
TableFileSchema
&
table_file
)
{
std
::
string
GetTableFileParentFolder
(
const
DBMetaOptions
&
options
,
const
meta
::
TableFileSchema
&
table_file
)
{
uint64_t
path_count
=
options
.
slave_paths_
.
size
()
+
1
;
std
::
string
target_path
=
options
.
path_
;
uint64_t
index
=
0
;
if
(
meta
::
TableFileSchema
::
NEW_INDEX
==
table_file
.
file_type_
)
{
if
(
meta
::
TableFileSchema
::
NEW_INDEX
==
table_file
.
file_type_
)
{
// index file is large file and to be persisted permanently
// we need to distribute index files to each db_path averagely
// round robin according to a file counter
...
...
@@ -67,17 +70,19 @@ std::string GetTableFileParentFolder(const DBMetaOptions& options, const meta::T
return
ConstructParentFolder
(
target_path
,
table_file
);
}
}
}
// namespace
long
GetMicroSecTimeStamp
()
{
int64_t
GetMicroSecTimeStamp
()
{
auto
now
=
std
::
chrono
::
system_clock
::
now
();
auto
micros
=
std
::
chrono
::
duration_cast
<
std
::
chrono
::
microseconds
>
(
now
.
time_since_epoch
()).
count
();
now
.
time_since_epoch
()).
count
();
return
micros
;
}
Status
CreateTablePath
(
const
DBMetaOptions
&
options
,
const
std
::
string
&
table_id
)
{
Status
CreateTablePath
(
const
DBMetaOptions
&
options
,
const
std
::
string
&
table_id
)
{
std
::
string
db_path
=
options
.
path_
;
std
::
string
table_path
=
db_path
+
TABLES_FOLDER
+
table_id
;
auto
status
=
server
::
CommonUtil
::
CreateDirectory
(
table_path
);
...
...
@@ -86,7 +91,7 @@ Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id
return
status
;
}
for
(
auto
&
path
:
options
.
slave_paths_
)
{
for
(
auto
&
path
:
options
.
slave_paths_
)
{
table_path
=
path
+
TABLES_FOLDER
+
table_id
;
status
=
server
::
CommonUtil
::
CreateDirectory
(
table_path
);
if
(
!
status
.
ok
())
{
...
...
@@ -98,17 +103,18 @@ Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id
return
Status
::
OK
();
}
Status
DeleteTablePath
(
const
DBMetaOptions
&
options
,
const
std
::
string
&
table_id
,
bool
force
)
{
Status
DeleteTablePath
(
const
DBMetaOptions
&
options
,
const
std
::
string
&
table_id
,
bool
force
)
{
std
::
vector
<
std
::
string
>
paths
=
options
.
slave_paths_
;
paths
.
push_back
(
options
.
path_
);
for
(
auto
&
path
:
paths
)
{
for
(
auto
&
path
:
paths
)
{
std
::
string
table_path
=
path
+
TABLES_FOLDER
+
table_id
;
if
(
force
)
{
if
(
force
)
{
boost
::
filesystem
::
remove_all
(
table_path
);
ENGINE_LOG_DEBUG
<<
"Remove table folder: "
<<
table_path
;
}
else
if
(
boost
::
filesystem
::
exists
(
table_path
)
&&
boost
::
filesystem
::
is_empty
(
table_path
))
{
}
else
if
(
boost
::
filesystem
::
exists
(
table_path
)
&&
boost
::
filesystem
::
is_empty
(
table_path
))
{
boost
::
filesystem
::
remove_all
(
table_path
);
ENGINE_LOG_DEBUG
<<
"Remove table folder: "
<<
table_path
;
}
...
...
@@ -117,7 +123,8 @@ Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id
return
Status
::
OK
();
}
Status
CreateTableFilePath
(
const
DBMetaOptions
&
options
,
meta
::
TableFileSchema
&
table_file
)
{
Status
CreateTableFilePath
(
const
DBMetaOptions
&
options
,
meta
::
TableFileSchema
&
table_file
)
{
std
::
string
parent_path
=
GetTableFileParentFolder
(
options
,
table_file
);
auto
status
=
server
::
CommonUtil
::
CreateDirectory
(
parent_path
);
...
...
@@ -131,17 +138,18 @@ Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema&
return
Status
::
OK
();
}
Status
GetTableFilePath
(
const
DBMetaOptions
&
options
,
meta
::
TableFileSchema
&
table_file
)
{
Status
GetTableFilePath
(
const
DBMetaOptions
&
options
,
meta
::
TableFileSchema
&
table_file
)
{
std
::
string
parent_path
=
ConstructParentFolder
(
options
.
path_
,
table_file
);
std
::
string
file_path
=
parent_path
+
"/"
+
table_file
.
file_id_
;
if
(
boost
::
filesystem
::
exists
(
file_path
))
{
if
(
boost
::
filesystem
::
exists
(
file_path
))
{
table_file
.
location_
=
file_path
;
return
Status
::
OK
();
}
else
{
for
(
auto
&
path
:
options
.
slave_paths_
)
{
for
(
auto
&
path
:
options
.
slave_paths_
)
{
parent_path
=
ConstructParentFolder
(
path
,
table_file
);
file_path
=
parent_path
+
"/"
+
table_file
.
file_id_
;
if
(
boost
::
filesystem
::
exists
(
file_path
))
{
if
(
boost
::
filesystem
::
exists
(
file_path
))
{
table_file
.
location_
=
file_path
;
return
Status
::
OK
();
}
...
...
@@ -155,49 +163,55 @@ Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& tab
return
Status
(
DB_ERROR
,
msg
);
}
Status
DeleteTableFilePath
(
const
DBMetaOptions
&
options
,
meta
::
TableFileSchema
&
table_file
)
{
Status
DeleteTableFilePath
(
const
DBMetaOptions
&
options
,
meta
::
TableFileSchema
&
table_file
)
{
utils
::
GetTableFilePath
(
options
,
table_file
);
boost
::
filesystem
::
remove
(
table_file
.
location_
);
return
Status
::
OK
();
}
bool
IsSameIndex
(
const
TableIndex
&
index1
,
const
TableIndex
&
index2
)
{
bool
IsSameIndex
(
const
TableIndex
&
index1
,
const
TableIndex
&
index2
)
{
return
index1
.
engine_type_
==
index2
.
engine_type_
&&
index1
.
nlist_
==
index2
.
nlist_
&&
index1
.
metric_type_
==
index2
.
metric_type_
;
}
meta
::
DateT
GetDate
(
const
std
::
time_t
&
t
,
int
day_delta
)
{
meta
::
DateT
GetDate
(
const
std
::
time_t
&
t
,
int
day_delta
)
{
struct
tm
ltm
;
localtime_r
(
&
t
,
&
ltm
);
if
(
day_delta
>
0
)
{
do
{
++
ltm
.
tm_mday
;
--
day_delta
;
}
while
(
day_delta
>
0
);
}
while
(
day_delta
>
0
);
mktime
(
&
ltm
);
}
else
if
(
day_delta
<
0
)
{
do
{
--
ltm
.
tm_mday
;
++
day_delta
;
}
while
(
day_delta
<
0
);
}
while
(
day_delta
<
0
);
mktime
(
&
ltm
);
}
else
{
ltm
.
tm_mday
;
}
return
ltm
.
tm_year
*
10000
+
ltm
.
tm_mon
*
100
+
ltm
.
tm_mday
;
return
ltm
.
tm_year
*
10000
+
ltm
.
tm_mon
*
100
+
ltm
.
tm_mday
;
}
meta
::
DateT
GetDateWithDelta
(
int
day_delta
)
{
meta
::
DateT
GetDateWithDelta
(
int
day_delta
)
{
return
GetDate
(
std
::
time
(
nullptr
),
day_delta
);
}
meta
::
DateT
GetDate
()
{
meta
::
DateT
GetDate
()
{
return
GetDate
(
std
::
time
(
nullptr
),
0
);
}
// URI format: dialect://username:password@host:port/database
Status
ParseMetaUri
(
const
std
::
string
&
uri
,
MetaUriInfo
&
info
)
{
Status
ParseMetaUri
(
const
std
::
string
&
uri
,
MetaUriInfo
&
info
)
{
std
::
string
dialect_regex
=
"(.*)"
;
std
::
string
username_tegex
=
"(.*)"
;
std
::
string
password_regex
=
"(.*)"
;
...
...
@@ -205,7 +219,7 @@ Status ParseMetaUri(const std::string& uri, MetaUriInfo& info) {
std
::
string
port_regex
=
"(.*)"
;
std
::
string
db_name_regex
=
"(.*)"
;
std
::
string
uri_regex_str
=
dialect_regex
+
"
\\
:
\\
/
\\
/"
+
dialect_regex
+
"
\\
:
\\
/
\\
/"
+
username_tegex
+
"
\\
:"
+
password_regex
+
"
\\
@"
+
host_regex
+
"
\\
:"
+
...
...
cpp/src/db/Utils.h
浏览文件 @
a0e4d19e
...
...
@@ -29,20 +29,30 @@ namespace milvus {
namespace
engine
{
namespace
utils
{
long
GetMicroSecTimeStamp
();
int64_t
GetMicroSecTimeStamp
();
Status
CreateTablePath
(
const
DBMetaOptions
&
options
,
const
std
::
string
&
table_id
);
Status
DeleteTablePath
(
const
DBMetaOptions
&
options
,
const
std
::
string
&
table_id
,
bool
force
=
true
);
Status
CreateTablePath
(
const
DBMetaOptions
&
options
,
const
std
::
string
&
table_id
);
Status
DeleteTablePath
(
const
DBMetaOptions
&
options
,
const
std
::
string
&
table_id
,
bool
force
=
true
);
Status
CreateTableFilePath
(
const
DBMetaOptions
&
options
,
meta
::
TableFileSchema
&
table_file
);
Status
GetTableFilePath
(
const
DBMetaOptions
&
options
,
meta
::
TableFileSchema
&
table_file
);
Status
DeleteTableFilePath
(
const
DBMetaOptions
&
options
,
meta
::
TableFileSchema
&
table_file
);
Status
CreateTableFilePath
(
const
DBMetaOptions
&
options
,
meta
::
TableFileSchema
&
table_file
);
Status
GetTableFilePath
(
const
DBMetaOptions
&
options
,
meta
::
TableFileSchema
&
table_file
);
Status
DeleteTableFilePath
(
const
DBMetaOptions
&
options
,
meta
::
TableFileSchema
&
table_file
);
bool
IsSameIndex
(
const
TableIndex
&
index1
,
const
TableIndex
&
index2
);
bool
IsSameIndex
(
const
TableIndex
&
index1
,
const
TableIndex
&
index2
);
meta
::
DateT
GetDate
(
const
std
::
time_t
&
t
,
int
day_delta
=
0
);
meta
::
DateT
GetDate
();
meta
::
DateT
GetDateWithDelta
(
int
day_delta
);
meta
::
DateT
GetDate
(
const
std
::
time_t
&
t
,
int
day_delta
=
0
);
meta
::
DateT
GetDate
();
meta
::
DateT
GetDateWithDelta
(
int
day_delta
);
struct
MetaUriInfo
{
std
::
string
dialect_
;
...
...
@@ -53,7 +63,8 @@ struct MetaUriInfo {
std
::
string
db_name_
;
};
Status
ParseMetaUri
(
const
std
::
string
&
uri
,
MetaUriInfo
&
info
);
Status
ParseMetaUri
(
const
std
::
string
&
uri
,
MetaUriInfo
&
info
);
}
// namespace utils
}
// namespace engine
...
...
cpp/src/db/engine/EngineFactory.cpp
浏览文件 @
a0e4d19e
...
...
@@ -15,10 +15,12 @@
// specific language governing permissions and limitations
// under the License.
#include "EngineFactory.h"
#include "ExecutionEngineImpl.h"
#include "
db/engine/
EngineFactory.h"
#include "
db/engine/
ExecutionEngineImpl.h"
#include "utils/Log.h"
#include <memory>
namespace
zilliz
{
namespace
milvus
{
namespace
engine
{
...
...
@@ -29,20 +31,19 @@ EngineFactory::Build(uint16_t dimension,
EngineType
index_type
,
MetricType
metric_type
,
int32_t
nlist
)
{
if
(
index_type
==
EngineType
::
INVALID
)
{
if
(
index_type
==
EngineType
::
INVALID
)
{
ENGINE_LOG_ERROR
<<
"Unsupported engine type"
;
return
nullptr
;
}
ENGINE_LOG_DEBUG
<<
"EngineFactory index type: "
<<
(
int
)
index_type
;
ENGINE_LOG_DEBUG
<<
"EngineFactory index type: "
<<
(
int
)
index_type
;
ExecutionEnginePtr
execution_engine_ptr
=
std
::
make_shared
<
ExecutionEngineImpl
>
(
dimension
,
location
,
index_type
,
metric_type
,
nlist
);
std
::
make_shared
<
ExecutionEngineImpl
>
(
dimension
,
location
,
index_type
,
metric_type
,
nlist
);
execution_engine_ptr
->
Init
();
return
execution_engine_ptr
;
}
}
}
}
\ No newline at end of file
}
// namespace engine
}
// namespace milvus
}
// namespace zilliz
cpp/src/db/engine/EngineFactory.h
浏览文件 @
a0e4d19e
...
...
@@ -21,19 +21,22 @@
#include "ExecutionEngine.h"
#include "utils/Status.h"
#include <string>
namespace
zilliz
{
namespace
milvus
{
namespace
engine
{
class
EngineFactory
{
public:
public:
static
ExecutionEnginePtr
Build
(
uint16_t
dimension
,
const
std
::
string
&
location
,
const
std
::
string
&
location
,
EngineType
index_type
,
MetricType
metric_type
,
int32_t
nlist
);
};
}
}
}
}
// namespace engine
}
// namespace milvus
}
// namespace zilliz
cpp/src/db/engine/ExecutionEngine.h
浏览文件 @
a0e4d19e
...
...
@@ -21,6 +21,7 @@
#include <vector>
#include <memory>
#include <string>
namespace
zilliz
{
namespace
milvus
{
...
...
@@ -41,9 +42,8 @@ enum class MetricType {
};
class
ExecutionEngine
{
public:
virtual
Status
AddWithIds
(
long
n
,
const
float
*
xdata
,
const
long
*
xids
)
=
0
;
public:
virtual
Status
AddWithIds
(
int64_t
n
,
const
float
*
xdata
,
const
int64_t
*
xids
)
=
0
;
virtual
size_t
Count
()
const
=
0
;
...
...
@@ -63,16 +63,16 @@ public:
virtual
std
::
shared_ptr
<
ExecutionEngine
>
Clone
()
=
0
;
virtual
Status
Merge
(
const
std
::
string
&
location
)
=
0
;
virtual
Status
Merge
(
const
std
::
string
&
location
)
=
0
;
virtual
Status
Search
(
long
n
,
const
float
*
data
,
long
k
,
long
nprobe
,
float
*
distances
,
long
*
labels
)
const
=
0
;
virtual
Status
Search
(
int64_t
n
,
const
float
*
data
,
int64_t
k
,
int64_t
nprobe
,
float
*
distances
,
int64_t
*
labels
)
const
=
0
;
virtual
std
::
shared_ptr
<
ExecutionEngine
>
BuildIndex
(
const
std
::
string
&
location
,
EngineType
engine_type
)
=
0
;
virtual
std
::
shared_ptr
<
ExecutionEngine
>
BuildIndex
(
const
std
::
string
&
location
,
EngineType
engine_type
)
=
0
;
virtual
Status
Cache
()
=
0
;
...
...
@@ -89,7 +89,6 @@ public:
using
ExecutionEnginePtr
=
std
::
shared_ptr
<
ExecutionEngine
>
;
}
// namespace engine
}
// namespace milvus
}
// namespace zilliz
cpp/src/db/engine/ExecutionEngineImpl.cpp
浏览文件 @
a0e4d19e
...
...
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#include "ExecutionEngineImpl.h"
#include "
db/engine/
ExecutionEngineImpl.h"
#include "cache/GpuCacheMgr.h"
#include "cache/CpuCacheMgr.h"
#include "metrics/Metrics.h"
...
...
@@ -29,6 +29,7 @@
#include "server/Config.h"
#include <stdexcept>
#include <utility>
namespace
zilliz
{
namespace
milvus
{
...
...
@@ -72,7 +73,8 @@ ExecutionEngineImpl::ExecutionEngineImpl(VecIndexPtr index,
nlist_
(
nlist
)
{
}
VecIndexPtr
ExecutionEngineImpl
::
CreatetVecIndex
(
EngineType
type
)
{
VecIndexPtr
ExecutionEngineImpl
::
CreatetVecIndex
(
EngineType
type
)
{
std
::
shared_ptr
<
VecIndex
>
index
;
switch
(
type
)
{
case
EngineType
::
FAISS_IDMAP
:
{
...
...
@@ -99,41 +101,48 @@ VecIndexPtr ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
return
index
;
}
Status
ExecutionEngineImpl
::
AddWithIds
(
long
n
,
const
float
*
xdata
,
const
long
*
xids
)
{
Status
ExecutionEngineImpl
::
AddWithIds
(
int64_t
n
,
const
float
*
xdata
,
const
int64_t
*
xids
)
{
auto
status
=
index_
->
Add
(
n
,
xdata
,
xids
);
return
status
;
}
size_t
ExecutionEngineImpl
::
Count
()
const
{
if
(
index_
==
nullptr
)
{
size_t
ExecutionEngineImpl
::
Count
()
const
{
if
(
index_
==
nullptr
)
{
ENGINE_LOG_ERROR
<<
"ExecutionEngineImpl: index is null, return count 0"
;
return
0
;
}
return
index_
->
Count
();
}
size_t
ExecutionEngineImpl
::
Size
()
const
{
size_t
ExecutionEngineImpl
::
Size
()
const
{
return
(
size_t
)
(
Count
()
*
Dimension
())
*
sizeof
(
float
);
}
size_t
ExecutionEngineImpl
::
Dimension
()
const
{
if
(
index_
==
nullptr
)
{
size_t
ExecutionEngineImpl
::
Dimension
()
const
{
if
(
index_
==
nullptr
)
{
ENGINE_LOG_ERROR
<<
"ExecutionEngineImpl: index is null, return dimension "
<<
dim_
;
return
dim_
;
}
return
index_
->
Dimension
();
}
size_t
ExecutionEngineImpl
::
PhysicalSize
()
const
{
size_t
ExecutionEngineImpl
::
PhysicalSize
()
const
{
return
server
::
CommonUtil
::
GetFileSize
(
location_
);
}
Status
ExecutionEngineImpl
::
Serialize
()
{
Status
ExecutionEngineImpl
::
Serialize
()
{
auto
status
=
write_index
(
index_
,
location_
);
return
status
;
}
Status
ExecutionEngineImpl
::
Load
(
bool
to_cache
)
{
Status
ExecutionEngineImpl
::
Load
(
bool
to_cache
)
{
index_
=
cache
::
CpuCacheMgr
::
GetInstance
()
->
GetIndex
(
location_
);
bool
already_in_cache
=
(
index_
!=
nullptr
);
if
(
!
already_in_cache
)
{
...
...
@@ -141,7 +150,7 @@ Status ExecutionEngineImpl::Load(bool to_cache) {
double
physical_size
=
PhysicalSize
();
server
::
CollectExecutionEngineMetrics
metrics
(
physical_size
);
index_
=
read_index
(
location_
);
if
(
index_
==
nullptr
)
{
if
(
index_
==
nullptr
)
{
std
::
string
msg
=
"Failed to load index from "
+
location_
;
ENGINE_LOG_ERROR
<<
msg
;
return
Status
(
DB_ERROR
,
msg
);
...
...
@@ -160,13 +169,14 @@ Status ExecutionEngineImpl::Load(bool to_cache) {
return
Status
::
OK
();
}
Status
ExecutionEngineImpl
::
CopyToGpu
(
uint64_t
device_id
)
{
Status
ExecutionEngineImpl
::
CopyToGpu
(
uint64_t
device_id
)
{
auto
index
=
cache
::
GpuCacheMgr
::
GetInstance
(
device_id
)
->
GetIndex
(
location_
);
bool
already_in_cache
=
(
index
!=
nullptr
);
if
(
already_in_cache
)
{
index_
=
index
;
}
else
{
if
(
index_
==
nullptr
)
{
if
(
index_
==
nullptr
)
{
ENGINE_LOG_ERROR
<<
"ExecutionEngineImpl: index is null, failed to copy to gpu"
;
return
Status
(
DB_ERROR
,
"index is null"
);
}
...
...
@@ -187,13 +197,14 @@ Status ExecutionEngineImpl::CopyToGpu(uint64_t device_id) {
return
Status
::
OK
();
}
Status
ExecutionEngineImpl
::
CopyToCpu
()
{
Status
ExecutionEngineImpl
::
CopyToCpu
()
{
auto
index
=
cache
::
CpuCacheMgr
::
GetInstance
()
->
GetIndex
(
location_
);
bool
already_in_cache
=
(
index
!=
nullptr
);
if
(
already_in_cache
)
{
index_
=
index
;
}
else
{
if
(
index_
==
nullptr
)
{
if
(
index_
==
nullptr
)
{
ENGINE_LOG_ERROR
<<
"ExecutionEngineImpl: index is null, failed to copy to cpu"
;
return
Status
(
DB_ERROR
,
"index is null"
);
}
...
...
@@ -213,8 +224,9 @@ Status ExecutionEngineImpl::CopyToCpu() {
return
Status
::
OK
();
}
ExecutionEnginePtr
ExecutionEngineImpl
::
Clone
()
{
if
(
index_
==
nullptr
)
{
ExecutionEnginePtr
ExecutionEngineImpl
::
Clone
()
{
if
(
index_
==
nullptr
)
{
ENGINE_LOG_ERROR
<<
"ExecutionEngineImpl: index is null, failed to clone"
;
return
nullptr
;
}
...
...
@@ -225,7 +237,8 @@ ExecutionEnginePtr ExecutionEngineImpl::Clone() {
return
ret
;
}
Status
ExecutionEngineImpl
::
Merge
(
const
std
::
string
&
location
)
{
Status
ExecutionEngineImpl
::
Merge
(
const
std
::
string
&
location
)
{
if
(
location
==
location_
)
{
return
Status
(
DB_ERROR
,
"Cannot Merge Self"
);
}
...
...
@@ -243,7 +256,7 @@ Status ExecutionEngineImpl::Merge(const std::string &location) {
}
}
if
(
index_
==
nullptr
)
{
if
(
index_
==
nullptr
)
{
ENGINE_LOG_ERROR
<<
"ExecutionEngineImpl: index is null, failed to merge"
;
return
Status
(
DB_ERROR
,
"index is null"
);
}
...
...
@@ -264,7 +277,7 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t
ENGINE_LOG_DEBUG
<<
"Build index file: "
<<
location
<<
" from: "
<<
location_
;
auto
from_index
=
std
::
dynamic_pointer_cast
<
BFIndex
>
(
index_
);
if
(
from_index
==
nullptr
)
{
if
(
from_index
==
nullptr
)
{
ENGINE_LOG_ERROR
<<
"ExecutionEngineImpl: from_index is null, failed to build index"
;
return
nullptr
;
}
...
...
@@ -282,21 +295,22 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t
AutoGenParams
(
to_index
->
GetType
(),
Count
(),
build_cfg
);
auto
status
=
to_index
->
BuildAll
(
Count
(),
from_index
->
GetRawVectors
(),
from_index
->
GetRawIds
(),
build_cfg
);
from_index
->
GetRawVectors
(),
from_index
->
GetRawIds
(),
build_cfg
);
if
(
!
status
.
ok
())
{
throw
Exception
(
DB_ERROR
,
status
.
message
());
}
return
std
::
make_shared
<
ExecutionEngineImpl
>
(
to_index
,
location
,
engine_type
,
metric_type_
,
nlist_
);
}
Status
ExecutionEngineImpl
::
Search
(
long
n
,
const
float
*
data
,
long
k
,
long
nprobe
,
float
*
distances
,
long
*
labels
)
const
{
if
(
index_
==
nullptr
)
{
Status
ExecutionEngineImpl
::
Search
(
int64_t
n
,
const
float
*
data
,
int64_t
k
,
int64_t
nprobe
,
float
*
distances
,
int64_t
*
labels
)
const
{
if
(
index_
==
nullptr
)
{
ENGINE_LOG_ERROR
<<
"ExecutionEngineImpl: index is null, failed to search"
;
return
Status
(
DB_ERROR
,
"index is null"
);
}
...
...
@@ -310,14 +324,16 @@ Status ExecutionEngineImpl::Search(long n,
return
status
;
}
Status
ExecutionEngineImpl
::
Cache
()
{
Status
ExecutionEngineImpl
::
Cache
()
{
cache
::
DataObjPtr
obj
=
std
::
make_shared
<
cache
::
DataObj
>
(
index_
,
PhysicalSize
());
zilliz
::
milvus
::
cache
::
CpuCacheMgr
::
GetInstance
()
->
InsertItem
(
location_
,
obj
);
return
Status
::
OK
();
}
Status
ExecutionEngineImpl
::
GpuCache
(
uint64_t
gpu_id
)
{
Status
ExecutionEngineImpl
::
GpuCache
(
uint64_t
gpu_id
)
{
cache
::
DataObjPtr
obj
=
std
::
make_shared
<
cache
::
DataObj
>
(
index_
,
PhysicalSize
());
zilliz
::
milvus
::
cache
::
GpuCacheMgr
::
GetInstance
(
gpu_id
)
->
InsertItem
(
location_
,
obj
);
...
...
@@ -325,8 +341,8 @@ Status ExecutionEngineImpl::GpuCache(uint64_t gpu_id) {
}
// TODO(linxj): remove.
Status
ExecutionEngineImpl
::
Init
()
{
using
namespace
zilliz
::
milvus
::
server
;
Status
ExecutionEngineImpl
::
Init
()
{
server
::
Config
&
config
=
server
::
Config
::
GetInstance
();
Status
s
=
config
.
GetDBConfigBuildIndexGPU
(
gpu_num_
);
if
(
!
s
.
ok
())
return
s
;
...
...
@@ -334,7 +350,6 @@ Status ExecutionEngineImpl::Init() {
return
Status
::
OK
();
}
}
// namespace engine
}
// namespace milvus
}
// namespace zilliz
cpp/src/db/engine/ExecutionEngineImpl.h
浏览文件 @
a0e4d19e
...
...
@@ -23,15 +23,12 @@
#include <memory>
#include <string>
namespace
zilliz
{
namespace
milvus
{
namespace
engine
{
class
ExecutionEngineImpl
:
public
ExecutionEngine
{
public:
public:
ExecutionEngineImpl
(
uint16_t
dimension
,
const
std
::
string
&
location
,
EngineType
index_type
,
...
...
@@ -44,7 +41,7 @@ public:
MetricType
metric_type
,
int32_t
nlist
);
Status
AddWithIds
(
long
n
,
const
float
*
xdata
,
const
long
*
xids
)
override
;
Status
AddWithIds
(
int64_t
n
,
const
float
*
xdata
,
const
int64_t
*
xids
)
override
;
size_t
Count
()
const
override
;
...
...
@@ -66,12 +63,12 @@ public:
Status
Merge
(
const
std
::
string
&
location
)
override
;
Status
Search
(
long
n
,
Status
Search
(
int64_t
n
,
const
float
*
data
,
long
k
,
long
nprobe
,
int64_t
k
,
int64_t
nprobe
,
float
*
distances
,
long
*
labels
)
const
override
;
int64_t
*
labels
)
const
override
;
ExecutionEnginePtr
BuildIndex
(
const
std
::
string
&
location
,
EngineType
engine_type
)
override
;
...
...
@@ -81,18 +78,24 @@ public:
Status
Init
()
override
;
EngineType
IndexEngineType
()
const
override
{
return
index_type_
;
}
EngineType
IndexEngineType
()
const
override
{
return
index_type_
;
}
MetricType
IndexMetricType
()
const
override
{
return
metric_type_
;
}
MetricType
IndexMetricType
()
const
override
{
return
metric_type_
;
}
std
::
string
GetLocation
()
const
override
{
return
location_
;
}
std
::
string
GetLocation
()
const
override
{
return
location_
;
}
private:
private:
VecIndexPtr
CreatetVecIndex
(
EngineType
type
);
VecIndexPtr
Load
(
const
std
::
string
&
location
);
protected:
protected:
VecIndexPtr
index_
=
nullptr
;
EngineType
index_type_
;
MetricType
metric_type_
;
...
...
@@ -104,7 +107,6 @@ protected:
int32_t
gpu_num_
=
0
;
};
}
// namespace engine
}
// namespace milvus
}
// namespace zilliz
cpp/src/db/insert/MemManager.h
浏览文件 @
a0e4d19e
...
...
@@ -23,6 +23,7 @@
#include <set>
#include <memory>
#include <string>
namespace
zilliz
{
namespace
milvus
{
...
...
@@ -30,7 +31,6 @@ namespace engine {
class
MemManager
{
public:
virtual
Status
InsertVectors
(
const
std
::
string
&
table_id
,
size_t
n
,
const
float
*
vectors
,
IDNumbers
&
vector_ids
)
=
0
;
...
...
@@ -43,11 +43,10 @@ class MemManager {
virtual
size_t
GetCurrentImmutableMem
()
=
0
;
virtual
size_t
GetCurrentMem
()
=
0
;
};
// MemManagerAbstract
using
MemManagerPtr
=
std
::
shared_ptr
<
MemManager
>
;
}
// namespace engine
}
// namespace milvus
}
// namespace zilliz
\ No newline at end of file
}
// namespace zilliz
cpp/src/db/insert/MemManagerImpl.cpp
浏览文件 @
a0e4d19e
...
...
@@ -16,19 +16,19 @@
// under the License.
#include "MemManagerImpl.h"
#include "
db/insert/
MemManagerImpl.h"
#include "VectorSource.h"
#include "utils/Log.h"
#include "db/Constants.h"
#include <thread>
namespace
zilliz
{
namespace
milvus
{
namespace
engine
{
MemTablePtr
MemManagerImpl
::
GetMemByTable
(
const
std
::
string
&
table_id
)
{
MemTablePtr
MemManagerImpl
::
GetMemByTable
(
const
std
::
string
&
table_id
)
{
auto
memIt
=
mem_id_map_
.
find
(
table_id
);
if
(
memIt
!=
mem_id_map_
.
end
())
{
return
memIt
->
second
;
...
...
@@ -38,11 +38,11 @@ MemTablePtr MemManagerImpl::GetMemByTable(const std::string &table_id) {
return
mem_id_map_
[
table_id
];
}
Status
MemManagerImpl
::
InsertVectors
(
const
std
::
string
&
table_id_
,
size_t
n
_
,
const
float
*
vectors
_
,
IDNumbers
&
vector_ids_
)
{
Status
MemManagerImpl
::
InsertVectors
(
const
std
::
string
&
table_id
_
,
size_t
n
_
,
const
float
*
vectors_
,
IDNumbers
&
vector_ids_
)
{
while
(
GetCurrentMem
()
>
options_
.
insert_buffer_size_
)
{
std
::
this_thread
::
sleep_for
(
std
::
chrono
::
milliseconds
(
1
));
}
...
...
@@ -52,11 +52,11 @@ Status MemManagerImpl::InsertVectors(const std::string &table_id_,
return
InsertVectorsNoLock
(
table_id_
,
n_
,
vectors_
,
vector_ids_
);
}
Status
MemManagerImpl
::
InsertVectorsNoLock
(
const
std
::
string
&
table_id
,
size_t
n
,
const
float
*
vectors
,
IDNumbers
&
vector_ids
)
{
Status
MemManagerImpl
::
InsertVectorsNoLock
(
const
std
::
string
&
table_id
,
size_t
n
,
const
float
*
vectors
,
IDNumbers
&
vector_ids
)
{
MemTablePtr
mem
=
GetMemByTable
(
table_id
);
VectorSourcePtr
source
=
std
::
make_shared
<
VectorSource
>
(
n
,
vectors
);
...
...
@@ -69,10 +69,11 @@ Status MemManagerImpl::InsertVectorsNoLock(const std::string &table_id,
return
status
;
}
Status
MemManagerImpl
::
ToImmutable
()
{
Status
MemManagerImpl
::
ToImmutable
()
{
std
::
unique_lock
<
std
::
mutex
>
lock
(
mutex_
);
MemIdMap
temp_map
;
for
(
auto
&
kv
:
mem_id_map_
)
{
for
(
auto
&
kv
:
mem_id_map_
)
{
if
(
kv
.
second
->
Empty
())
{
//empty table, no need to serialize
temp_map
.
insert
(
kv
);
...
...
@@ -85,7 +86,8 @@ Status MemManagerImpl::ToImmutable() {
return
Status
::
OK
();
}
Status
MemManagerImpl
::
Serialize
(
std
::
set
<
std
::
string
>
&
table_ids
)
{
Status
MemManagerImpl
::
Serialize
(
std
::
set
<
std
::
string
>
&
table_ids
)
{
ToImmutable
();
std
::
unique_lock
<
std
::
mutex
>
lock
(
serialization_mtx_
);
table_ids
.
clear
();
...
...
@@ -97,7 +99,8 @@ Status MemManagerImpl::Serialize(std::set<std::string> &table_ids) {
return
Status
::
OK
();
}
Status
MemManagerImpl
::
EraseMemVector
(
const
std
::
string
&
table_id
)
{
Status
MemManagerImpl
::
EraseMemVector
(
const
std
::
string
&
table_id
)
{
{
//erase MemVector from rapid-insert cache
std
::
unique_lock
<
std
::
mutex
>
lock
(
mutex_
);
mem_id_map_
.
erase
(
table_id
);
...
...
@@ -117,7 +120,8 @@ Status MemManagerImpl::EraseMemVector(const std::string &table_id) {
return
Status
::
OK
();
}
size_t
MemManagerImpl
::
GetCurrentMutableMem
()
{
size_t
MemManagerImpl
::
GetCurrentMutableMem
()
{
size_t
total_mem
=
0
;
for
(
auto
&
kv
:
mem_id_map_
)
{
auto
memTable
=
kv
.
second
;
...
...
@@ -126,7 +130,8 @@ size_t MemManagerImpl::GetCurrentMutableMem() {
return
total_mem
;
}
size_t
MemManagerImpl
::
GetCurrentImmutableMem
()
{
size_t
MemManagerImpl
::
GetCurrentImmutableMem
()
{
size_t
total_mem
=
0
;
for
(
auto
&
mem_table
:
immu_mem_list_
)
{
total_mem
+=
mem_table
->
GetCurrentMem
();
...
...
@@ -134,10 +139,11 @@ size_t MemManagerImpl::GetCurrentImmutableMem() {
return
total_mem
;
}
size_t
MemManagerImpl
::
GetCurrentMem
()
{
size_t
MemManagerImpl
::
GetCurrentMem
()
{
return
GetCurrentMutableMem
()
+
GetCurrentImmutableMem
();
}
}
// namespace engine
}
// namespace milvus
}
// namespace zilliz
\ No newline at end of file
}
// namespace zilliz
cpp/src/db/insert/MemManagerImpl.h
浏览文件 @
a0e4d19e
...
...
@@ -24,12 +24,13 @@
#include "utils/Status.h"
#include <map>
#include <set>
#include <vector>
#include <string>
#include <ctime>
#include <memory>
#include <mutex>
namespace
zilliz
{
namespace
milvus
{
namespace
engine
{
...
...
@@ -39,7 +40,8 @@ class MemManagerImpl : public MemManager {
using
Ptr
=
std
::
shared_ptr
<
MemManagerImpl
>
;
MemManagerImpl
(
const
meta
::
MetaPtr
&
meta
,
const
DBOptions
&
options
)
:
meta_
(
meta
),
options_
(
options
)
{}
:
meta_
(
meta
),
options_
(
options
)
{
}
Status
InsertVectors
(
const
std
::
string
&
table_id
,
size_t
n
,
const
float
*
vectors
,
IDNumbers
&
vector_ids
)
override
;
...
...
@@ -71,7 +73,6 @@ class MemManagerImpl : public MemManager {
std
::
mutex
serialization_mtx_
;
};
// NewMemManager
}
// namespace engine
}
// namespace milvus
}
// namespace zilliz
\ No newline at end of file
}
// namespace zilliz
cpp/src/db/insert/MemMenagerFactory.cpp
浏览文件 @
a0e4d19e
...
...
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#include "MemMenagerFactory.h"
#include "
db/insert/
MemMenagerFactory.h"
#include "MemManagerImpl.h"
#include "utils/Log.h"
#include "utils/Exception.h"
...
...
@@ -26,12 +26,14 @@
#include <cstdlib>
#include <string>
#include <regex>
#include <memory>
namespace
zilliz
{
namespace
milvus
{
namespace
engine
{
MemManagerPtr
MemManagerFactory
::
Build
(
const
std
::
shared_ptr
<
meta
::
Meta
>&
meta
,
const
DBOptions
&
options
)
{
MemManagerPtr
MemManagerFactory
::
Build
(
const
std
::
shared_ptr
<
meta
::
Meta
>
&
meta
,
const
DBOptions
&
options
)
{
return
std
::
make_shared
<
MemManagerImpl
>
(
meta
,
options
);
}
...
...
cpp/src/db/insert/MemMenagerFactory.h
浏览文件 @
a0e4d19e
...
...
@@ -20,15 +20,17 @@
#include "MemManager.h"
#include "db/meta/Meta.h"
#include <memory>
namespace
zilliz
{
namespace
milvus
{
namespace
engine
{
class
MemManagerFactory
{
public:
public:
static
MemManagerPtr
Build
(
const
std
::
shared_ptr
<
meta
::
Meta
>
&
meta
,
const
DBOptions
&
options
);
};
}
}
}
}
// namespace engine
}
// namespace milvus
}
// namespace zilliz
cpp/src/db/insert/MemTable.cpp
浏览文件 @
a0e4d19e
...
...
@@ -16,9 +16,11 @@
// under the License.
#include "MemTable.h"
#include "
db/insert/
MemTable.h"
#include "utils/Log.h"
#include <memory>
#include <string>
namespace
zilliz
{
namespace
milvus
{
...
...
@@ -30,13 +32,11 @@ MemTable::MemTable(const std::string &table_id,
table_id_
(
table_id
),
meta_
(
meta
),
options_
(
options
)
{
}
Status
MemTable
::
Add
(
VectorSourcePtr
&
source
,
IDNumbers
&
vector_ids
)
{
Status
MemTable
::
Add
(
VectorSourcePtr
&
source
,
IDNumbers
&
vector_ids
)
{
while
(
!
source
->
AllAdded
())
{
MemTableFilePtr
current_mem_table_file
;
if
(
!
mem_table_file_list_
.
empty
())
{
current_mem_table_file
=
mem_table_file_list_
.
back
();
...
...
@@ -62,15 +62,18 @@ Status MemTable::Add(VectorSourcePtr &source, IDNumbers &vector_ids) {
return
Status
::
OK
();
}
void
MemTable
::
GetCurrentMemTableFile
(
MemTableFilePtr
&
mem_table_file
)
{
void
MemTable
::
GetCurrentMemTableFile
(
MemTableFilePtr
&
mem_table_file
)
{
mem_table_file
=
mem_table_file_list_
.
back
();
}
size_t
MemTable
::
GetTableFileCount
()
{
size_t
MemTable
::
GetTableFileCount
()
{
return
mem_table_file_list_
.
size
();
}
Status
MemTable
::
Serialize
()
{
Status
MemTable
::
Serialize
()
{
for
(
auto
mem_table_file
=
mem_table_file_list_
.
begin
();
mem_table_file
!=
mem_table_file_list_
.
end
();)
{
auto
status
=
(
*
mem_table_file
)
->
Serialize
();
if
(
!
status
.
ok
())
{
...
...
@@ -84,15 +87,18 @@ Status MemTable::Serialize() {
return
Status
::
OK
();
}
bool
MemTable
::
Empty
()
{
bool
MemTable
::
Empty
()
{
return
mem_table_file_list_
.
empty
();
}
const
std
::
string
&
MemTable
::
GetTableId
()
const
{
const
std
::
string
&
MemTable
::
GetTableId
()
const
{
return
table_id_
;
}
size_t
MemTable
::
GetCurrentMem
()
{
size_t
MemTable
::
GetCurrentMem
()
{
std
::
lock_guard
<
std
::
mutex
>
lock
(
mutex_
);
size_t
total_mem
=
0
;
for
(
auto
&
mem_table_file
:
mem_table_file_list_
)
{
...
...
@@ -103,4 +109,4 @@ size_t MemTable::GetCurrentMem() {
}
// namespace engine
}
// namespace milvus
}
// namespace zilliz
\ No newline at end of file
}
// namespace zilliz
cpp/src/db/insert/MemTable.h
浏览文件 @
a0e4d19e
...
...
@@ -23,7 +23,9 @@
#include "utils/Status.h"
#include <mutex>
#include <vector>
#include <memory>
#include <string>
namespace
zilliz
{
namespace
milvus
{
...
...
@@ -59,11 +61,10 @@ class MemTable {
DBOptions
options_
;
std
::
mutex
mutex_
;
};
//MemTable
using
MemTablePtr
=
std
::
shared_ptr
<
MemTable
>
;
}
// namespace engine
}
// namespace milvus
}
// namespace zilliz
\ No newline at end of file
}
// namespace zilliz
cpp/src/db/insert/MemTableFile.cpp
浏览文件 @
a0e4d19e
...
...
@@ -16,14 +16,14 @@
// under the License.
#include "MemTableFile.h"
#include "
db/insert/
MemTableFile.h"
#include "db/Constants.h"
#include "db/engine/EngineFactory.h"
#include "metrics/Metrics.h"
#include "utils/Log.h"
#include <cmath>
#include <string>
namespace
zilliz
{
namespace
milvus
{
...
...
@@ -35,20 +35,19 @@ MemTableFile::MemTableFile(const std::string &table_id,
table_id_
(
table_id
),
meta_
(
meta
),
options_
(
options
)
{
current_mem_
=
0
;
auto
status
=
CreateTableFile
();
if
(
status
.
ok
())
{
execution_engine_
=
EngineFactory
::
Build
(
table_file_schema_
.
dimension_
,
table_file_schema_
.
location_
,
(
EngineType
)
table_file_schema_
.
engine_type_
,
(
MetricType
)
table_file_schema_
.
metric_type_
,
(
MetricType
)
table_file_schema_
.
metric_type_
,
table_file_schema_
.
nlist_
);
}
}
Status
MemTableFile
::
CreateTableFile
()
{
Status
MemTableFile
::
CreateTableFile
()
{
meta
::
TableFileSchema
table_file_schema
;
table_file_schema
.
table_id_
=
table_id_
;
auto
status
=
meta_
->
CreateTableFile
(
table_file_schema
);
...
...
@@ -61,8 +60,8 @@ Status MemTableFile::CreateTableFile() {
return
status
;
}
Status
MemTableFile
::
Add
(
const
VectorSourcePtr
&
source
,
IDNumbers
&
vector_ids
)
{
Status
MemTableFile
::
Add
(
const
VectorSourcePtr
&
source
,
IDNumbers
&
vector_ids
)
{
if
(
table_file_schema_
.
dimension_
<=
0
)
{
std
::
string
err_msg
=
"MemTableFile::Add: table_file_schema dimension = "
+
std
::
to_string
(
table_file_schema_
.
dimension_
)
+
", table_id = "
+
table_file_schema_
.
table_id_
;
...
...
@@ -75,7 +74,8 @@ Status MemTableFile::Add(const VectorSourcePtr &source, IDNumbers& vector_ids) {
if
(
mem_left
>=
single_vector_mem_size
)
{
size_t
num_vectors_to_add
=
std
::
ceil
(
mem_left
/
single_vector_mem_size
);
size_t
num_vectors_added
;
auto
status
=
source
->
Add
(
execution_engine_
,
table_file_schema_
,
num_vectors_to_add
,
num_vectors_added
,
vector_ids
);
auto
status
=
source
->
Add
(
execution_engine_
,
table_file_schema_
,
num_vectors_to_add
,
num_vectors_added
,
vector_ids
);
if
(
status
.
ok
())
{
current_mem_
+=
(
num_vectors_added
*
single_vector_mem_size
);
}
...
...
@@ -84,20 +84,24 @@ Status MemTableFile::Add(const VectorSourcePtr &source, IDNumbers& vector_ids) {
return
Status
::
OK
();
}
size_t
MemTableFile
::
GetCurrentMem
()
{
size_t
MemTableFile
::
GetCurrentMem
()
{
return
current_mem_
;
}
size_t
MemTableFile
::
GetMemLeft
()
{
size_t
MemTableFile
::
GetMemLeft
()
{
return
(
MAX_TABLE_FILE_MEM
-
current_mem_
);
}
bool
MemTableFile
::
IsFull
()
{
bool
MemTableFile
::
IsFull
()
{
size_t
single_vector_mem_size
=
table_file_schema_
.
dimension_
*
VECTOR_TYPE_SIZE
;
return
(
GetMemLeft
()
<
single_vector_mem_size
);
}
Status
MemTableFile
::
Serialize
()
{
Status
MemTableFile
::
Serialize
()
{
size_t
size
=
GetCurrentMem
();
server
::
CollectSerializeMetrics
metrics
(
size
);
...
...
@@ -107,7 +111,7 @@ Status MemTableFile::Serialize() {
//if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size
//else set file type to RAW, no need to build index
if
(
table_file_schema_
.
engine_type_
!=
(
int
)
EngineType
::
FAISS_IDMAP
)
{
if
(
table_file_schema_
.
engine_type_
!=
(
int
)
EngineType
::
FAISS_IDMAP
)
{
table_file_schema_
.
file_type_
=
(
size
>=
table_file_schema_
.
index_file_size_
)
?
meta
::
TableFileSchema
::
TO_INDEX
:
meta
::
TableFileSchema
::
RAW
;
}
else
{
...
...
@@ -117,9 +121,9 @@ Status MemTableFile::Serialize() {
auto
status
=
meta_
->
UpdateTableFile
(
table_file_schema_
);
ENGINE_LOG_DEBUG
<<
"New "
<<
((
table_file_schema_
.
file_type_
==
meta
::
TableFileSchema
::
RAW
)
?
"raw"
:
"to_index"
)
<<
" file "
<<
table_file_schema_
.
file_id_
<<
" of size "
<<
size
<<
" bytes"
;
<<
" file "
<<
table_file_schema_
.
file_id_
<<
" of size "
<<
size
<<
" bytes"
;
if
(
options_
.
insert_cache_immediately_
)
{
if
(
options_
.
insert_cache_immediately_
)
{
execution_engine_
->
Cache
();
}
...
...
@@ -128,4 +132,4 @@ Status MemTableFile::Serialize() {
}
// namespace engine
}
// namespace milvus
}
// namespace zilliz
\ No newline at end of file
}
// namespace zilliz
cpp/src/db/insert/MemTableFile.h
浏览文件 @
a0e4d19e
...
...
@@ -23,17 +23,18 @@
#include "db/engine/ExecutionEngine.h"
#include "utils/Status.h"
#include <string>
#include <memory>
namespace
zilliz
{
namespace
milvus
{
namespace
engine
{
class
MemTableFile
{
public:
MemTableFile
(
const
std
::
string
&
table_id
,
const
meta
::
MetaPtr
&
meta
,
const
DBOptions
&
options
);
Status
Add
(
const
VectorSourcePtr
&
source
,
IDNumbers
&
vector_ids
);
Status
Add
(
const
VectorSourcePtr
&
source
,
IDNumbers
&
vector_ids
);
size_t
GetCurrentMem
();
...
...
@@ -44,25 +45,20 @@ class MemTableFile {
Status
Serialize
();
private:
Status
CreateTableFile
();
private:
const
std
::
string
table_id_
;
meta
::
TableFileSchema
table_file_schema_
;
meta
::
MetaPtr
meta_
;
DBOptions
options_
;
size_t
current_mem_
;
ExecutionEnginePtr
execution_engine_
;
};
//MemTableFile
using
MemTableFilePtr
=
std
::
shared_ptr
<
MemTableFile
>
;
}
// namespace engine
}
// namespace milvus
}
// namespace zilliz
\ No newline at end of file
}
// namespace zilliz
cpp/src/db/insert/VectorSource.cpp
浏览文件 @
a0e4d19e
...
...
@@ -16,32 +16,30 @@
// under the License.
#include "VectorSource.h"
#include "
db/insert/
VectorSource.h"
#include "db/engine/ExecutionEngine.h"
#include "db/engine/EngineFactory.h"
#include "utils/Log.h"
#include "metrics/Metrics.h"
namespace
zilliz
{
namespace
milvus
{
namespace
engine
{
VectorSource
::
VectorSource
(
const
size_t
&
n
,
const
float
*
vectors
)
:
n_
(
n
),
vectors_
(
vectors
),
id_generator_
(
std
::
make_shared
<
SimpleIDGenerator
>
())
{
n_
(
n
),
vectors_
(
vectors
),
id_generator_
(
std
::
make_shared
<
SimpleIDGenerator
>
())
{
current_num_vectors_added
=
0
;
}
Status
VectorSource
::
Add
(
const
ExecutionEnginePtr
&
execution_engine
,
const
meta
::
TableFileSchema
&
table_file_schema
,
const
size_t
&
num_vectors_to_add
,
size_t
&
num_vectors_adde
d
,
IDNumbers
&
vector_ids
)
{
Status
VectorSource
::
Add
(
const
ExecutionEnginePtr
&
execution_engine
,
const
meta
::
TableFileSchema
&
table_file_schema
,
const
size_t
&
num_vectors_to_ad
d
,
size_t
&
num_vectors_added
,
IDNumbers
&
vector_ids
)
{
server
::
CollectAddMetrics
metrics
(
n_
,
table_file_schema
.
dimension_
);
num_vectors_added
=
current_num_vectors_added
+
num_vectors_to_add
<=
n_
?
...
...
@@ -52,7 +50,7 @@ Status VectorSource::Add(const ExecutionEnginePtr &execution_engine,
}
else
{
vector_ids_to_add
.
resize
(
num_vectors_added
);
for
(
int
pos
=
current_num_vectors_added
;
pos
<
current_num_vectors_added
+
num_vectors_added
;
pos
++
)
{
vector_ids_to_add
[
pos
-
current_num_vectors_added
]
=
vector_ids
[
pos
];
vector_ids_to_add
[
pos
-
current_num_vectors_added
]
=
vector_ids
[
pos
];
}
}
Status
status
=
execution_engine
->
AddWithIds
(
num_vectors_added
,
...
...
@@ -70,18 +68,21 @@ Status VectorSource::Add(const ExecutionEnginePtr &execution_engine,
return
status
;
}
size_t
VectorSource
::
GetNumVectorsAdded
()
{
size_t
VectorSource
::
GetNumVectorsAdded
()
{
return
current_num_vectors_added
;
}
bool
VectorSource
::
AllAdded
()
{
bool
VectorSource
::
AllAdded
()
{
return
(
current_num_vectors_added
==
n_
);
}
IDNumbers
VectorSource
::
GetVectorIds
()
{
IDNumbers
VectorSource
::
GetVectorIds
()
{
return
vector_ids_
;
}
}
// namespace engine
}
// namespace milvus
}
// namespace zilliz
\ No newline at end of file
}
// namespace zilliz
cpp/src/db/insert/VectorSource.h
浏览文件 @
a0e4d19e
...
...
@@ -23,6 +23,7 @@
#include "db/engine/ExecutionEngine.h"
#include "utils/Status.h"
#include <memory>
namespace
zilliz
{
namespace
milvus
{
...
...
@@ -45,7 +46,6 @@ class VectorSource {
IDNumbers
GetVectorIds
();
private:
const
size_t
n_
;
const
float
*
vectors_
;
IDNumbers
vector_ids_
;
...
...
@@ -53,11 +53,10 @@ class VectorSource {
size_t
current_num_vectors_added
;
std
::
shared_ptr
<
IDGenerator
>
id_generator_
;
};
//VectorSource
using
VectorSourcePtr
=
std
::
shared_ptr
<
VectorSource
>
;
}
// namespace engine
}
// namespace milvus
}
// namespace zilliz
\ No newline at end of file
}
// namespace zilliz
cpp/src/db/meta/Meta.h
浏览文件 @
a0e4d19e
...
...
@@ -25,14 +25,16 @@
#include <cstddef>
#include <memory>
#include <vector>
#include <string>
namespace
zilliz
{
namespace
milvus
{
namespace
engine
{
namespace
meta
{
static
const
char
*
META_TABLES
=
"Tables"
;
static
const
char
*
META_TABLEFILES
=
"TableFiles"
;
static
const
char
*
META_TABLES
=
"Tables"
;
static
const
char
*
META_TABLEFILES
=
"TableFiles"
;
class
Meta
{
public:
...
...
@@ -46,7 +48,7 @@ class Meta {
virtual
Status
AllTables
(
std
::
vector
<
TableSchema
>
&
table_schema_array
)
=
0
;
virtual
Status
UpdateTableIndex
(
const
std
::
string
&
table_id
,
const
TableIndex
&
index
)
=
0
;
virtual
Status
UpdateTableIndex
(
const
std
::
string
&
table_id
,
const
TableIndex
&
index
)
=
0
;
virtual
Status
UpdateTableFlag
(
const
std
::
string
&
table_id
,
int64_t
flag
)
=
0
;
...
...
@@ -83,9 +85,9 @@ class Meta {
virtual
Status
FilesByType
(
const
std
::
string
&
table_id
,
const
std
::
vector
<
int
>
&
file_types
,
std
::
vector
<
std
::
string
>
&
file_ids
)
=
0
;
std
::
vector
<
std
::
string
>
&
file_ids
)
=
0
;
virtual
Status
DescribeTableIndex
(
const
std
::
string
&
table_id
,
TableIndex
&
index
)
=
0
;
virtual
Status
DescribeTableIndex
(
const
std
::
string
&
table_id
,
TableIndex
&
index
)
=
0
;
virtual
Status
DropTableIndex
(
const
std
::
string
&
table_id
)
=
0
;
...
...
@@ -96,7 +98,6 @@ class Meta {
virtual
Status
DropAll
()
=
0
;
virtual
Status
Count
(
const
std
::
string
&
table_id
,
uint64_t
&
result
)
=
0
;
};
// MetaData
using
MetaPtr
=
std
::
shared_ptr
<
Meta
>
;
...
...
cpp/src/db/meta/MetaConsts.h
浏览文件 @
a0e4d19e
...
...
@@ -23,20 +23,20 @@ namespace engine {
namespace
meta
{
const
size_t
K
=
1024UL
;
const
size_t
M
=
K
*
K
;
const
size_t
G
=
K
*
M
;
const
size_t
T
=
K
*
G
;
const
size_t
M
=
K
*
K
;
const
size_t
G
=
K
*
M
;
const
size_t
T
=
K
*
G
;
const
size_t
S_PS
=
1UL
;
const
size_t
MS_PS
=
1000
*
S_PS
;
const
size_t
US_PS
=
1000
*
MS_PS
;
const
size_t
NS_PS
=
1000
*
US_PS
;
const
size_t
MS_PS
=
1000
*
S_PS
;
const
size_t
US_PS
=
1000
*
MS_PS
;
const
size_t
NS_PS
=
1000
*
US_PS
;
const
size_t
SECOND
=
1UL
;
const
size_t
M_SEC
=
60
*
SECOND
;
const
size_t
H_SEC
=
60
*
M_SEC
;
const
size_t
D_SEC
=
24
*
H_SEC
;
const
size_t
W_SEC
=
7
*
D_SEC
;
const
size_t
M_SEC
=
60
*
SECOND
;
const
size_t
H_SEC
=
60
*
M_SEC
;
const
size_t
D_SEC
=
24
*
H_SEC
;
const
size_t
W_SEC
=
7
*
D_SEC
;
}
// namespace meta
}
// namespace engine
...
...
cpp/src/db/meta/MetaFactory.cpp
浏览文件 @
a0e4d19e
...
...
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#include "MetaFactory.h"
#include "
db/meta/
MetaFactory.h"
#include "SqliteMetaImpl.h"
#include "MySQLMetaImpl.h"
#include "utils/Log.h"
...
...
@@ -28,46 +28,51 @@
#include <cstdlib>
#include <string>
#include <string.h>
#include <memory>
namespace
zilliz
{
namespace
milvus
{
namespace
engine
{
DBMetaOptions
MetaFactory
::
BuildOption
(
const
std
::
string
&
path
)
{
auto
p
=
path
;
if
(
p
==
""
)
{
srand
(
time
(
nullptr
));
std
::
stringstream
ss
;
ss
<<
"/tmp/"
<<
rand
();
p
=
ss
.
str
();
}
DBMetaOptions
meta
;
meta
.
path_
=
p
;
return
meta
;
DBMetaOptions
MetaFactory
::
BuildOption
(
const
std
::
string
&
path
)
{
auto
p
=
path
;
if
(
p
==
""
)
{
srand
(
time
(
nullptr
));
std
::
stringstream
ss
;
uint32_t
rd
=
0
;
rand_r
(
&
rd
);
ss
<<
"/tmp/"
<<
rd
;
p
=
ss
.
str
();
}
meta
::
MetaPtr
MetaFactory
::
Build
(
const
DBMetaOptions
&
metaOptions
,
const
int
&
mode
)
{
std
::
string
uri
=
metaOptions
.
backend_uri_
;
DBMetaOptions
meta
;
meta
.
path_
=
p
;
return
meta
;
}
meta
::
MetaPtr
MetaFactory
::
Build
(
const
DBMetaOptions
&
metaOptions
,
const
int
&
mode
)
{
std
::
string
uri
=
metaOptions
.
backend_uri_
;
utils
::
MetaUriInfo
uri_info
;
auto
status
=
utils
::
ParseMetaUri
(
uri
,
uri_info
);
if
(
!
status
.
ok
())
{
ENGINE_LOG_ERROR
<<
"Wrong URI format: URI = "
<<
uri
;
throw
InvalidArgumentException
(
"Wrong URI format "
);
}
utils
::
MetaUriInfo
uri_info
;
auto
status
=
utils
::
ParseMetaUri
(
uri
,
uri_info
);
if
(
!
status
.
ok
())
{
ENGINE_LOG_ERROR
<<
"Wrong URI format: URI = "
<<
uri
;
throw
InvalidArgumentException
(
"Wrong URI format "
);
}
if
(
strcasecmp
(
uri_info
.
dialect_
.
c_str
(),
"mysql"
)
==
0
)
{
ENGINE_LOG_INFO
<<
"Using MySQL"
;
return
std
::
make_shared
<
meta
::
MySQLMetaImpl
>
(
metaOptions
,
mode
);
}
else
if
(
strcasecmp
(
uri_info
.
dialect_
.
c_str
(),
"sqlite"
)
==
0
)
{
ENGINE_LOG_INFO
<<
"Using SQLite"
;
return
std
::
make_shared
<
meta
::
SqliteMetaImpl
>
(
metaOptions
);
}
else
{
ENGINE_LOG_ERROR
<<
"Invalid dialect in URI: dialect = "
<<
uri_info
.
dialect_
;
throw
InvalidArgumentException
(
"URI dialect is not mysql / sqlite"
);
}
if
(
strcasecmp
(
uri_info
.
dialect_
.
c_str
(),
"mysql"
)
==
0
)
{
ENGINE_LOG_INFO
<<
"Using MySQL"
;
return
std
::
make_shared
<
meta
::
MySQLMetaImpl
>
(
metaOptions
,
mode
);
}
else
if
(
strcasecmp
(
uri_info
.
dialect_
.
c_str
(),
"sqlite"
)
==
0
)
{
ENGINE_LOG_INFO
<<
"Using SQLite"
;
return
std
::
make_shared
<
meta
::
SqliteMetaImpl
>
(
metaOptions
);
}
else
{
ENGINE_LOG_ERROR
<<
"Invalid dialect in URI: dialect = "
<<
uri_info
.
dialect_
;
throw
InvalidArgumentException
(
"URI dialect is not mysql / sqlite"
);
}
}
}
// namespace engine
}
// namespace milvus
...
...
cpp/src/db/meta/MetaFactory.h
浏览文件 @
a0e4d19e
...
...
@@ -20,18 +20,19 @@
#include "Meta.h"
#include "db/Options.h"
#include <string>
namespace
zilliz
{
namespace
milvus
{
namespace
engine
{
class
MetaFactory
{
public:
public:
static
DBMetaOptions
BuildOption
(
const
std
::
string
&
path
=
""
);
static
meta
::
MetaPtr
Build
(
const
DBMetaOptions
&
metaOptions
,
const
int
&
mode
);
};
}
}
}
}
// namespace engine
}
// namespace milvus
}
// namespace zilliz
cpp/src/db/meta/MetaTypes.h
浏览文件 @
a0e4d19e
...
...
@@ -23,21 +23,22 @@
#include <vector>
#include <map>
#include <string>
#include <memory>
namespace
zilliz
{
namespace
milvus
{
namespace
engine
{
namespace
meta
{
constexpr
int32_t
DEFAULT_ENGINE_TYPE
=
(
int
)
EngineType
::
FAISS_IDMAP
;
constexpr
int32_t
DEFAULT_ENGINE_TYPE
=
(
int
)
EngineType
::
FAISS_IDMAP
;
constexpr
int32_t
DEFAULT_NLIST
=
16384
;
constexpr
int32_t
DEFAULT_METRIC_TYPE
=
(
int
)
MetricType
::
L2
;
constexpr
int32_t
DEFAULT_METRIC_TYPE
=
(
int
)
MetricType
::
L2
;
constexpr
int32_t
DEFAULT_INDEX_FILE_SIZE
=
ONE_GB
;
constexpr
int64_t
FLAG_MASK_NO_USERID
=
0x1
;
constexpr
int64_t
FLAG_MASK_HAS_USERID
=
0x1
<<
1
;
constexpr
int64_t
FLAG_MASK_HAS_USERID
=
0x1
<<
1
;
using
DateT
=
int
;
using
DateT
=
int
;
const
DateT
EmptyDate
=
-
1
;
using
DatesT
=
std
::
vector
<
DateT
>
;
...
...
@@ -49,7 +50,7 @@ struct TableSchema {
size_t
id_
=
0
;
std
::
string
table_id_
;
int32_t
state_
=
(
int
)
NORMAL
;
int32_t
state_
=
(
int
)
NORMAL
;
uint16_t
dimension_
=
0
;
int64_t
created_on_
=
0
;
int64_t
flag_
=
0
;
...
...
cpp/src/db/meta/MySQLConnectionPool.cpp
浏览文件 @
a0e4d19e
...
...
@@ -16,37 +16,39 @@
// under the License.
#include "MySQLConnectionPool.h"
#include "
db/meta/
MySQLConnectionPool.h"
namespace
zilliz
{
namespace
milvus
{
namespace
engine
{
namespace
meta
{
// Do a simple form of in-use connection limiting: wait to return
// a connection until there are a reasonably low number in use
// already. Can't do this in create() because we're interested in
// connections actually in use, not those created. Also note that
// we keep our own count; ConnectionPool::size() isn't the same!
mysqlpp
::
Connection
*
MySQLConnectionPool
::
grab
()
{
while
(
conns_in_use_
>
max_pool_size_
)
{
sleep
(
1
);
}
++
conns_in_use_
;
return
mysqlpp
::
ConnectionPool
::
grab
();
// Do a simple form of in-use connection limiting: wait to return
// a connection until there are a reasonably low number in use
// already. Can't do this in create() because we're interested in
// connections actually in use, not those created. Also note that
// we keep our own count; ConnectionPool::size() isn't the same!
mysqlpp
::
Connection
*
MySQLConnectionPool
::
grab
()
{
while
(
conns_in_use_
>
max_pool_size_
)
{
sleep
(
1
);
}
// Other half of in-use conn count limit
void
MySQLConnectionPool
::
release
(
const
mysqlpp
::
Connection
*
pc
)
{
mysqlpp
::
ConnectionPool
::
release
(
pc
);
++
conns_in_use_
;
return
mysqlpp
::
ConnectionPool
::
grab
();
}
if
(
conns_in_use_
<=
0
)
{
ENGINE_LOG_WARNING
<<
"MySQLConnetionPool::release: conns_in_use_ is less than zero. conns_in_use_ = "
<<
conns_in_use_
;
}
else
{
--
conns_in_use_
;
}
// Other half of in-use conn count limit
void
MySQLConnectionPool
::
release
(
const
mysqlpp
::
Connection
*
pc
)
{
mysqlpp
::
ConnectionPool
::
release
(
pc
);
if
(
conns_in_use_
<=
0
)
{
ENGINE_LOG_WARNING
<<
"MySQLConnetionPool::release: conns_in_use_ is less than zero. conns_in_use_ = "
<<
conns_in_use_
;
}
else
{
--
conns_in_use_
;
}
}
// int MySQLConnectionPool::getConnectionsInUse() {
// return conns_in_use_;
...
...
@@ -56,39 +58,42 @@ namespace meta {
// max_idle_time_ = max_idle;
// }
std
::
string
MySQLConnectionPool
::
getDB
()
{
return
db_
;
}
// Superclass overrides
mysqlpp
::
Connection
*
MySQLConnectionPool
::
create
()
{
std
::
string
MySQLConnectionPool
::
getDB
()
{
return
db_
;
}
try
{
// Create connection using the parameters we were passed upon
// creation.
mysqlpp
::
Connection
*
conn
=
new
mysqlpp
::
Connection
();
conn
->
set_option
(
new
mysqlpp
::
ReconnectOption
(
true
));
conn
->
connect
(
db_
.
empty
()
?
0
:
db_
.
c_str
(),
server_
.
empty
()
?
0
:
server_
.
c_str
(),
user_
.
empty
()
?
0
:
user_
.
c_str
(),
password_
.
empty
()
?
0
:
password_
.
c_str
(),
port_
);
return
conn
;
}
catch
(
const
mysqlpp
::
ConnectionFailed
&
er
)
{
ENGINE_LOG_ERROR
<<
"Failed to connect to database server"
<<
": "
<<
er
.
what
();
return
nullptr
;
}
// Superclass overrides
mysqlpp
::
Connection
*
MySQLConnectionPool
::
create
()
{
try
{
// Create connection using the parameters we were passed upon
// creation.
mysqlpp
::
Connection
*
conn
=
new
mysqlpp
::
Connection
();
conn
->
set_option
(
new
mysqlpp
::
ReconnectOption
(
true
));
conn
->
connect
(
db_
.
empty
()
?
0
:
db_
.
c_str
(),
server_
.
empty
()
?
0
:
server_
.
c_str
(),
user_
.
empty
()
?
0
:
user_
.
c_str
(),
password_
.
empty
()
?
0
:
password_
.
c_str
(),
port_
);
return
conn
;
}
catch
(
const
mysqlpp
::
ConnectionFailed
&
er
)
{
ENGINE_LOG_ERROR
<<
"Failed to connect to database server"
<<
": "
<<
er
.
what
();
return
nullptr
;
}
}
void
MySQLConnectionPool
::
destroy
(
mysqlpp
::
Connection
*
cp
)
{
// Our superclass can't know how we created the Connection, so
// it delegates destruction to us, to be safe.
delete
cp
;
}
void
MySQLConnectionPool
::
destroy
(
mysqlpp
::
Connection
*
cp
)
{
// Our superclass can't know how we created the Connection, so
// it delegates destruction to us, to be safe.
delete
cp
;
}
unsigned
int
MySQLConnectionPool
::
max_idle_time
()
{
return
max_idle_time_
;
}
unsigned
int
MySQLConnectionPool
::
max_idle_time
()
{
return
max_idle_time_
;
}
}
// namespace meta
}
// namespace engine
...
...
cpp/src/db/meta/MySQLConnectionPool.h
浏览文件 @
a0e4d19e
...
...
@@ -30,8 +30,7 @@ namespace engine {
namespace
meta
{
class
MySQLConnectionPool
:
public
mysqlpp
::
ConnectionPool
{
public:
public:
// The object's only constructor
MySQLConnectionPool
(
std
::
string
dbName
,
std
::
string
userName
,
...
...
@@ -39,15 +38,13 @@ public:
std
::
string
serverIp
,
int
port
=
0
,
int
maxPoolSize
=
8
)
:
db_
(
dbName
),
user_
(
userName
),
password_
(
passWord
),
server_
(
serverIp
),
port_
(
port
),
max_pool_size_
(
maxPoolSize
)
{
db_
(
dbName
),
user_
(
userName
),
password_
(
passWord
),
server_
(
serverIp
),
port_
(
port
),
max_pool_size_
(
maxPoolSize
)
{
conns_in_use_
=
0
;
max_idle_time_
=
10
;
//10 seconds
}
...
...
@@ -68,8 +65,7 @@ public:
std
::
string
getDB
();
protected:
protected:
// Superclass overrides
mysqlpp
::
Connection
*
create
()
override
;
...
...
@@ -77,7 +73,7 @@ protected:
unsigned
int
max_idle_time
()
override
;
private:
private:
// Number of connections currently in use
std
::
atomic
<
int
>
conns_in_use_
;
...
...
@@ -93,4 +89,4 @@ private:
}
// namespace meta
}
// namespace engine
}
// namespace milvus
}
// namespace zilliz
\ No newline at end of file
}
// namespace zilliz
cpp/src/db/meta/MySQLMetaImpl.cpp
浏览文件 @
a0e4d19e
此差异已折叠。
点击以展开。
cpp/src/db/meta/MySQLMetaImpl.h
浏览文件 @
a0e4d19e
...
...
@@ -21,26 +21,25 @@
#include "db/Options.h"
#include "MySQLConnectionPool.h"
#include
"mysql++/mysql++.h"
#include
<mysql++/mysql++.h>
#include <mutex>
#include <vector>
#include <string>
#include <memory>
namespace
zilliz
{
namespace
milvus
{
namespace
engine
{
namespace
meta
{
// auto StoragePrototype(const std::string& path);
using
namespace
mysqlpp
;
class
MySQLMetaImpl
:
public
Meta
{
public:
MySQLMetaImpl
(
const
DBMetaOptions
&
options
_
,
const
int
&
mode
);
MySQLMetaImpl
(
const
DBMetaOptions
&
options
,
const
int
&
mode
);
~
MySQLMetaImpl
();
Status
CreateTable
(
TableSchema
&
table_schema
)
override
;
Status
DescribeTable
(
TableSchema
&
group_info_
)
override
;
Status
DescribeTable
(
TableSchema
&
table_schema
)
override
;
Status
HasTable
(
const
std
::
string
&
table_id
,
bool
&
has_or_not
)
override
;
...
...
@@ -63,11 +62,11 @@ class MySQLMetaImpl : public Meta {
const
std
::
vector
<
int
>
&
file_types
,
std
::
vector
<
std
::
string
>
&
file_ids
)
override
;
Status
UpdateTableIndex
(
const
std
::
string
&
table_id
,
const
TableIndex
&
index
)
override
;
Status
UpdateTableIndex
(
const
std
::
string
&
table_id
,
const
TableIndex
&
index
)
override
;
Status
UpdateTableFlag
(
const
std
::
string
&
table_id
,
int64_t
flag
)
override
;
Status
DescribeTableIndex
(
const
std
::
string
&
table_id
,
TableIndex
&
index
)
override
;
Status
DescribeTableIndex
(
const
std
::
string
&
table_id
,
TableIndex
&
index
)
override
;
Status
DropTableIndex
(
const
std
::
string
&
table_id
)
override
;
...
...
@@ -102,12 +101,12 @@ class MySQLMetaImpl : public Meta {
private:
Status
NextFileId
(
std
::
string
&
file_id
);
Status
NextTableId
(
std
::
string
&
table_id
);
Status
DiscardFiles
(
long
long
to_discard_size
);
Status
DiscardFiles
(
int64_t
to_discard_size
);
void
ValidateMetaSchema
();
Status
Initialize
();
private:
private:
const
DBMetaOptions
options_
;
const
int
mode_
;
...
...
cpp/src/db/meta/SqliteMetaImpl.cpp
浏览文件 @
a0e4d19e
此差异已折叠。
点击以展开。
cpp/src/db/meta/SqliteMetaImpl.h
浏览文件 @
a0e4d19e
...
...
@@ -21,22 +21,25 @@
#include "db/Options.h"
#include <mutex>
#include <vector>
#include <string>
namespace
zilliz
{
namespace
milvus
{
namespace
engine
{
namespace
meta
{
auto
StoragePrototype
(
const
std
::
string
&
path
);
auto
StoragePrototype
(
const
std
::
string
&
path
);
class
SqliteMetaImpl
:
public
Meta
{
public:
explicit
SqliteMetaImpl
(
const
DBMetaOptions
&
options
_
);
explicit
SqliteMetaImpl
(
const
DBMetaOptions
&
options
);
~
SqliteMetaImpl
();
Status
CreateTable
(
TableSchema
&
table_schema
)
override
;
Status
DescribeTable
(
TableSchema
&
group_info_
)
override
;
Status
DescribeTable
(
TableSchema
&
table_schema
)
override
;
Status
HasTable
(
const
std
::
string
&
table_id
,
bool
&
has_or_not
)
override
;
...
...
@@ -58,11 +61,11 @@ class SqliteMetaImpl : public Meta {
const
std
::
vector
<
int
>
&
file_types
,
std
::
vector
<
std
::
string
>
&
file_ids
)
override
;
Status
UpdateTableIndex
(
const
std
::
string
&
table_id
,
const
TableIndex
&
index
)
override
;
Status
UpdateTableIndex
(
const
std
::
string
&
table_id
,
const
TableIndex
&
index
)
override
;
Status
UpdateTableFlag
(
const
std
::
string
&
table_id
,
int64_t
flag
)
override
;
Status
DescribeTableIndex
(
const
std
::
string
&
table_id
,
TableIndex
&
index
)
override
;
Status
DescribeTableIndex
(
const
std
::
string
&
table_id
,
TableIndex
&
index
)
override
;
Status
DropTableIndex
(
const
std
::
string
&
table_id
)
override
;
...
...
@@ -96,12 +99,12 @@ class SqliteMetaImpl : public Meta {
private:
Status
NextFileId
(
std
::
string
&
file_id
);
Status
NextTableId
(
std
::
string
&
table_id
);
Status
DiscardFiles
(
long
to_discard_size
);
Status
DiscardFiles
(
int64_t
to_discard_size
);
void
ValidateMetaSchema
();
Status
Initialize
();
private:
private:
const
DBMetaOptions
options_
;
std
::
mutex
meta_mutex_
;
};
// DBMetaImpl
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录