Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
37fd7963
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
Fork自
milvus / milvus
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
37fd7963
编写于
3月 09, 2020
作者:
G
groot
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
#1571 Meta engine type become IDMAP after drop index for BINARY table
Signed-off-by:
N
groot
<
yihua.mo@zilliz.com
>
上级
037e4054
变更
14
隐藏空白更改
内联
并排
Showing
14 changed file
with
68 addition
and
67 deletion
+68
-67
CHANGELOG.md
CHANGELOG.md
+1
-0
core/src/db/DBImpl.cpp
core/src/db/DBImpl.cpp
+3
-3
core/src/db/Utils.cpp
core/src/db/Utils.cpp
+13
-0
core/src/db/Utils.h
core/src/db/Utils.h
+6
-0
core/src/db/engine/ExecutionEngineImpl.cpp
core/src/db/engine/ExecutionEngineImpl.cpp
+2
-3
core/src/db/meta/SqliteMetaImpl.cpp
core/src/db/meta/SqliteMetaImpl.cpp
+29
-33
core/src/scheduler/task/BuildIndexTask.cpp
core/src/scheduler/task/BuildIndexTask.cpp
+3
-2
core/src/scheduler/task/SearchTask.cpp
core/src/scheduler/task/SearchTask.cpp
+2
-2
core/src/server/delivery/request/CreateIndexRequest.cpp
core/src/server/delivery/request/CreateIndexRequest.cpp
+2
-1
core/src/server/delivery/request/CreateTableRequest.cpp
core/src/server/delivery/request/CreateTableRequest.cpp
+2
-1
core/src/server/delivery/request/InsertRequest.cpp
core/src/server/delivery/request/InsertRequest.cpp
+3
-2
core/src/server/delivery/request/SearchRequest.cpp
core/src/server/delivery/request/SearchRequest.cpp
+2
-1
core/src/utils/ValidationUtil.cpp
core/src/utils/ValidationUtil.cpp
+0
-13
core/src/utils/ValidationUtil.h
core/src/utils/ValidationUtil.h
+0
-6
未找到文件。
CHANGELOG.md
浏览文件 @
37fd7963
...
...
@@ -45,6 +45,7 @@ Please mark all change in change log and use the issue from GitHub
-
\#
1535 Degradation searching performance with metric_type: binary_idmap
-
\#
1556 Index file not created after table and index created
-
\#
1560 Search crashed with Super-high dimensional binary vector
-
\#
1571 Meta engine type become IDMAP after drop index for BINARY table
## Feature
-
\#
216 Add CLI to get server info
...
...
core/src/db/DBImpl.cpp
浏览文件 @
37fd7963
...
...
@@ -366,8 +366,8 @@ DBImpl::PreloadTable(const std::string& table_id) {
if
(
file
.
file_type_
==
meta
::
TableFileSchema
::
FILE_TYPE
::
RAW
||
file
.
file_type_
==
meta
::
TableFileSchema
::
FILE_TYPE
::
TO_INDEX
||
file
.
file_type_
==
meta
::
TableFileSchema
::
FILE_TYPE
::
BACKUP
)
{
engine_type
=
server
::
ValidationUtil
::
IsBinaryMetricType
(
file
.
metric_type_
)
?
EngineType
::
FAISS_BIN_IDMAP
:
EngineType
::
FAISS_IDMAP
;
engine_type
=
utils
::
IsBinaryMetricType
(
file
.
metric_type_
)
?
EngineType
::
FAISS_BIN_IDMAP
:
EngineType
::
FAISS_IDMAP
;
}
else
{
engine_type
=
(
EngineType
)
file
.
engine_type_
;
}
...
...
@@ -1019,7 +1019,7 @@ DBImpl::GetVectorByIdHelper(const std::string& table_id, IDNumber vector_id, Vec
auto
deleted
=
std
::
find
(
deleted_docs
.
begin
(),
deleted_docs
.
end
(),
offset
);
if
(
deleted
==
deleted_docs
.
end
())
{
// Load raw vector
bool
is_binary
=
server
::
ValidationUtil
::
IsBinaryMetricType
(
file
.
metric_type_
);
bool
is_binary
=
utils
::
IsBinaryMetricType
(
file
.
metric_type_
);
size_t
single_vector_bytes
=
is_binary
?
file
.
dimension_
/
8
:
file
.
dimension_
*
sizeof
(
float
);
std
::
vector
<
uint8_t
>
raw_vector
;
status
=
segment_reader
.
LoadVectors
(
offset
*
single_vector_bytes
,
single_vector_bytes
,
raw_vector
);
...
...
core/src/db/Utils.cpp
浏览文件 @
37fd7963
...
...
@@ -220,6 +220,19 @@ IsRawIndexType(int32_t type) {
return
(
type
==
(
int32_t
)
EngineType
::
FAISS_IDMAP
)
||
(
type
==
(
int32_t
)
EngineType
::
FAISS_BIN_IDMAP
);
}
bool
IsBinaryIndexType
(
int32_t
index_type
)
{
return
(
index_type
==
(
int32_t
)
engine
::
EngineType
::
FAISS_BIN_IDMAP
)
||
(
index_type
==
(
int32_t
)
engine
::
EngineType
::
FAISS_BIN_IVFFLAT
);
}
bool
IsBinaryMetricType
(
int32_t
metric_type
)
{
return
(
metric_type
==
(
int32_t
)
engine
::
MetricType
::
HAMMING
)
||
(
metric_type
==
(
int32_t
)
engine
::
MetricType
::
JACCARD
)
||
(
metric_type
==
(
int32_t
)
engine
::
MetricType
::
TANIMOTO
);
}
meta
::
DateT
GetDate
(
const
std
::
time_t
&
t
,
int
day_delta
)
{
struct
tm
ltm
;
...
...
core/src/db/Utils.h
浏览文件 @
37fd7963
...
...
@@ -48,6 +48,12 @@ IsSameIndex(const TableIndex& index1, const TableIndex& index2);
bool
IsRawIndexType
(
int32_t
type
);
static
bool
IsBinaryIndexType
(
int32_t
index_type
);
bool
IsBinaryMetricType
(
int32_t
metric_type
);
meta
::
DateT
GetDate
(
const
std
::
time_t
&
t
,
int
day_delta
=
0
);
meta
::
DateT
...
...
core/src/db/engine/ExecutionEngineImpl.cpp
浏览文件 @
37fd7963
...
...
@@ -100,9 +100,8 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, const std::string&
index_type_
(
index_type
),
metric_type_
(
metric_type
),
index_params_
(
index_params
)
{
EngineType
tmp_index_type
=
server
::
ValidationUtil
::
IsBinaryMetricType
((
int32_t
)
metric_type
)
?
EngineType
::
FAISS_BIN_IDMAP
:
EngineType
::
FAISS_IDMAP
;
EngineType
tmp_index_type
=
utils
::
IsBinaryMetricType
((
int32_t
)
metric_type
)
?
EngineType
::
FAISS_BIN_IDMAP
:
EngineType
::
FAISS_IDMAP
;
index_
=
CreatetVecIndex
(
tmp_index_type
);
if
(
!
index_
)
{
throw
Exception
(
DB_ERROR
,
"Unsupported index type"
);
...
...
core/src/db/meta/SqliteMetaImpl.cpp
浏览文件 @
37fd7963
...
...
@@ -804,8 +804,18 @@ SqliteMetaImpl::DropTableIndex(const std::string& table_id) {
c
(
&
TableFileSchema
::
file_type_
)
==
(
int
)
TableFileSchema
::
BACKUP
));
// set table index type to raw
auto
groups
=
ConnectorPtr
->
select
(
columns
(
&
TableSchema
::
metric_type_
),
where
(
c
(
&
TableSchema
::
table_id_
)
==
table_id
));
int32_t
raw_engine_type
=
DEFAULT_ENGINE_TYPE
;
if
(
groups
.
size
()
==
1
)
{
int32_t
metric_type_
=
std
::
get
<
0
>
(
groups
[
0
]);
if
(
engine
::
utils
::
IsBinaryMetricType
(
metric_type_
))
{
raw_engine_type
=
(
int32_t
)
EngineType
::
FAISS_BIN_IDMAP
;
}
}
ConnectorPtr
->
update_all
(
set
(
c
(
&
TableSchema
::
engine_type_
)
=
DEFAULT_ENGINE_TYPE
,
c
(
&
TableSchema
::
index_params_
)
=
"{}"
),
set
(
c
(
&
TableSchema
::
engine_type_
)
=
raw_engine_type
,
c
(
&
TableSchema
::
index_params_
)
=
"{}"
),
where
(
c
(
&
TableSchema
::
table_id_
)
==
table_id
));
ENGINE_LOG_DEBUG
<<
"Successfully drop table index, table id = "
<<
table_id
;
...
...
@@ -1189,29 +1199,21 @@ SqliteMetaImpl::FilesByType(const std::string& table_id, const std::vector<int>&
file_schema
.
metric_type_
=
table_schema
.
metric_type_
;
switch
(
file_schema
.
file_type_
)
{
case
(
int
)
TableFileSchema
::
RAW
:
++
raw_count
;
break
;
case
(
int
)
TableFileSchema
::
NEW
:
++
new_count
;
case
(
int
)
TableFileSchema
::
RAW
:
++
raw_count
;
break
;
case
(
int
)
TableFileSchema
::
NEW_MERGE
:
++
new_merge_count
;
case
(
int
)
TableFileSchema
::
NEW
:
++
new_count
;
break
;
case
(
int
)
TableFileSchema
::
NEW_INDEX
:
++
new_index_count
;
case
(
int
)
TableFileSchema
::
NEW_MERGE
:
++
new_merge_count
;
break
;
case
(
int
)
TableFileSchema
::
TO_INDEX
:
++
to_index_count
;
case
(
int
)
TableFileSchema
::
NEW_INDEX
:
++
new_index_count
;
break
;
case
(
int
)
TableFileSchema
::
INDEX
:
++
index_count
;
case
(
int
)
TableFileSchema
::
TO_INDEX
:
++
to_index_count
;
break
;
case
(
int
)
TableFileSchema
::
BACKUP
:
++
backup_count
;
case
(
int
)
TableFileSchema
::
INDEX
:
++
index_count
;
break
;
default:
case
(
int
)
TableFileSchema
::
BACKUP
:
++
backup_count
;
break
;
default:
break
;
}
auto
status
=
utils
::
GetTableFilePath
(
options_
,
file_schema
);
...
...
@@ -1225,29 +1227,23 @@ SqliteMetaImpl::FilesByType(const std::string& table_id, const std::vector<int>&
std
::
string
msg
=
"Get table files by type."
;
for
(
int
file_type
:
file_types
)
{
switch
(
file_type
)
{
case
(
int
)
TableFileSchema
::
RAW
:
msg
=
msg
+
" raw files:"
+
std
::
to_string
(
raw_count
);
break
;
case
(
int
)
TableFileSchema
::
NEW
:
msg
=
msg
+
" new files:"
+
std
::
to_string
(
new_count
);
case
(
int
)
TableFileSchema
::
RAW
:
msg
=
msg
+
" raw files:"
+
std
::
to_string
(
raw_count
);
break
;
case
(
int
)
TableFileSchema
::
NEW_MERGE
:
msg
=
msg
+
" new_merge files:"
+
std
::
to_string
(
new_merge_count
);
case
(
int
)
TableFileSchema
::
NEW
:
msg
=
msg
+
" new files:"
+
std
::
to_string
(
new_count
);
break
;
case
(
int
)
TableFileSchema
::
NEW_
INDEX
:
msg
=
msg
+
" new_index files:"
+
std
::
to_string
(
new_index
_count
);
case
(
int
)
TableFileSchema
::
NEW_
MERGE
:
msg
=
msg
+
" new_merge files:"
+
std
::
to_string
(
new_merge
_count
);
break
;
case
(
int
)
TableFileSchema
::
TO_INDEX
:
msg
=
msg
+
" to_index files:"
+
std
::
to_string
(
to
_index_count
);
case
(
int
)
TableFileSchema
::
NEW_INDEX
:
msg
=
msg
+
" new_index files:"
+
std
::
to_string
(
new
_index_count
);
break
;
case
(
int
)
TableFileSchema
::
INDEX
:
msg
=
msg
+
" index files:"
+
std
::
to_string
(
index_count
);
case
(
int
)
TableFileSchema
::
TO_INDEX
:
msg
=
msg
+
" to_index files:"
+
std
::
to_string
(
to_index_count
);
break
;
case
(
int
)
TableFileSchema
::
BACKUP
:
msg
=
msg
+
" backup files:"
+
std
::
to_string
(
backup_count
);
case
(
int
)
TableFileSchema
::
INDEX
:
msg
=
msg
+
" index files:"
+
std
::
to_string
(
index_count
);
break
;
default:
case
(
int
)
TableFileSchema
::
BACKUP
:
msg
=
msg
+
" backup files:"
+
std
::
to_string
(
backup_count
);
break
;
default:
break
;
}
}
ENGINE_LOG_DEBUG
<<
msg
;
...
...
core/src/scheduler/task/BuildIndexTask.cpp
浏览文件 @
37fd7963
...
...
@@ -17,6 +17,7 @@
#include <thread>
#include <utility>
#include "db/Utils.h"
#include "db/engine/EngineFactory.h"
#include "metrics/Metrics.h"
#include "scheduler/job/BuildIndexJob.h"
...
...
@@ -35,8 +36,8 @@ XBuildIndexTask::XBuildIndexTask(TableFileSchemaPtr file, TaskLabelPtr label)
if
(
file
->
file_type_
==
TableFileSchema
::
FILE_TYPE
::
RAW
||
file
->
file_type_
==
TableFileSchema
::
FILE_TYPE
::
TO_INDEX
||
file
->
file_type_
==
TableFileSchema
::
FILE_TYPE
::
BACKUP
)
{
engine_type
=
server
::
ValidationUtil
::
IsBinaryMetricType
(
file
->
metric_type_
)
?
EngineType
::
FAISS_BIN_IDMAP
:
EngineType
::
FAISS_IDMAP
;
engine_type
=
engine
::
utils
::
IsBinaryMetricType
(
file
->
metric_type_
)
?
EngineType
::
FAISS_BIN_IDMAP
:
EngineType
::
FAISS_IDMAP
;
}
else
{
engine_type
=
(
EngineType
)
file
->
engine_type_
;
}
...
...
core/src/scheduler/task/SearchTask.cpp
浏览文件 @
37fd7963
...
...
@@ -109,8 +109,8 @@ XSearchTask::XSearchTask(const std::shared_ptr<server::Context>& context, TableF
if
(
file
->
file_type_
==
TableFileSchema
::
FILE_TYPE
::
RAW
||
file
->
file_type_
==
TableFileSchema
::
FILE_TYPE
::
TO_INDEX
||
file
->
file_type_
==
TableFileSchema
::
FILE_TYPE
::
BACKUP
)
{
engine_type
=
server
::
ValidationUtil
::
IsBinaryMetricType
(
file
->
metric_type_
)
?
EngineType
::
FAISS_BIN_IDMAP
:
EngineType
::
FAISS_IDMAP
;
engine_type
=
engine
::
utils
::
IsBinaryMetricType
(
file
->
metric_type_
)
?
EngineType
::
FAISS_BIN_IDMAP
:
EngineType
::
FAISS_IDMAP
;
}
else
{
engine_type
=
(
EngineType
)
file
->
engine_type_
;
}
...
...
core/src/server/delivery/request/CreateIndexRequest.cpp
浏览文件 @
37fd7963
...
...
@@ -10,6 +10,7 @@
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "server/delivery/request/CreateIndexRequest.h"
#include "db/Utils.h"
#include "server/Config.h"
#include "server/DBWrapper.h"
#include "utils/Log.h"
...
...
@@ -83,7 +84,7 @@ CreateIndexRequest::OnExecute() {
status
=
DBWrapper
::
DB
()
->
DescribeTable
(
table_info
);
int32_t
adapter_index_type
=
index_type_
;
if
(
ValidationUtil
::
IsBinaryMetricType
(
table_info
.
metric_type_
))
{
// binary vector not allow
if
(
engine
::
utils
::
IsBinaryMetricType
(
table_info
.
metric_type_
))
{
// binary vector not allow
if
(
adapter_index_type
==
static_cast
<
int32_t
>
(
engine
::
EngineType
::
FAISS_IDMAP
))
{
adapter_index_type
=
static_cast
<
int32_t
>
(
engine
::
EngineType
::
FAISS_BIN_IDMAP
);
}
else
if
(
adapter_index_type
==
static_cast
<
int32_t
>
(
engine
::
EngineType
::
FAISS_IVFFLAT
))
{
...
...
core/src/server/delivery/request/CreateTableRequest.cpp
浏览文件 @
37fd7963
...
...
@@ -10,6 +10,7 @@
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "server/delivery/request/CreateTableRequest.h"
#include "db/Utils.h"
#include "server/DBWrapper.h"
#include "server/delivery/request/BaseRequest.h"
#include "utils/Log.h"
...
...
@@ -78,7 +79,7 @@ CreateTableRequest::OnExecute() {
table_info
.
metric_type_
=
metric_type_
;
// some metric type only support binary vector, adapt the index type
if
(
ValidationUtil
::
IsBinaryMetricType
(
metric_type_
))
{
if
(
engine
::
utils
::
IsBinaryMetricType
(
metric_type_
))
{
if
(
table_info
.
engine_type_
==
static_cast
<
int32_t
>
(
engine
::
EngineType
::
FAISS_IDMAP
))
{
table_info
.
engine_type_
=
static_cast
<
int32_t
>
(
engine
::
EngineType
::
FAISS_BIN_IDMAP
);
}
else
if
(
table_info
.
engine_type_
==
static_cast
<
int32_t
>
(
engine
::
EngineType
::
FAISS_IVFFLAT
))
{
...
...
core/src/server/delivery/request/InsertRequest.cpp
浏览文件 @
37fd7963
...
...
@@ -10,6 +10,7 @@
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "server/delivery/request/InsertRequest.h"
#include "db/Utils.h"
#include "server/DBWrapper.h"
#include "utils/CommonUtil.h"
#include "utils/Log.h"
...
...
@@ -115,7 +116,7 @@ InsertRequest::OnExecute() {
#endif
// step 4: some metric type doesn't support float vectors
if
(
!
vectors_data_
.
float_data_
.
empty
())
{
// insert float vectors
if
(
ValidationUtil
::
IsBinaryMetricType
(
table_schema
.
metric_type_
))
{
if
(
engine
::
utils
::
IsBinaryMetricType
(
table_schema
.
metric_type_
))
{
return
Status
(
SERVER_INVALID_ROWRECORD_ARRAY
,
"Table metric type doesn't support float vectors."
);
}
...
...
@@ -131,7 +132,7 @@ InsertRequest::OnExecute() {
"The vector dimension must be equal to the table dimension."
);
}
}
else
if
(
!
vectors_data_
.
binary_data_
.
empty
())
{
// insert binary vectors
if
(
!
ValidationUtil
::
IsBinaryMetricType
(
table_schema
.
metric_type_
))
{
if
(
!
engine
::
utils
::
IsBinaryMetricType
(
table_schema
.
metric_type_
))
{
return
Status
(
SERVER_INVALID_ROWRECORD_ARRAY
,
"Table metric type doesn't support binary vectors."
);
}
...
...
core/src/server/delivery/request/SearchRequest.cpp
浏览文件 @
37fd7963
...
...
@@ -10,6 +10,7 @@
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "server/delivery/request/SearchRequest.h"
#include "db/Utils.h"
#include "server/DBWrapper.h"
#include "utils/CommonUtil.h"
#include "utils/Log.h"
...
...
@@ -103,7 +104,7 @@ SearchRequest::OnExecute() {
rc
.
RecordSection
(
"check validation"
);
// step 4: check metric type
if
(
ValidationUtil
::
IsBinaryMetricType
(
table_schema
.
metric_type_
))
{
if
(
engine
::
utils
::
IsBinaryMetricType
(
table_schema
.
metric_type_
))
{
// check prepared binary data
if
(
vectors_data_
.
binary_data_
.
size
()
%
vector_count
!=
0
)
{
return
Status
(
SERVER_INVALID_ROWRECORD_ARRAY
,
...
...
core/src/utils/ValidationUtil.cpp
浏览文件 @
37fd7963
...
...
@@ -264,12 +264,6 @@ ValidationUtil::ValidateSearchParams(const milvus::json& search_params, const en
return
Status
::
OK
();
}
bool
ValidationUtil
::
IsBinaryIndexType
(
int32_t
index_type
)
{
return
(
index_type
==
static_cast
<
int32_t
>
(
engine
::
EngineType
::
FAISS_BIN_IDMAP
))
||
(
index_type
==
static_cast
<
int32_t
>
(
engine
::
EngineType
::
FAISS_BIN_IVFFLAT
));
}
Status
ValidationUtil
::
ValidateTableIndexFileSize
(
int64_t
index_file_size
)
{
if
(
index_file_size
<=
0
||
index_file_size
>
INDEX_FILE_SIZE_LIMIT
)
{
...
...
@@ -294,13 +288,6 @@ ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) {
return
Status
::
OK
();
}
bool
ValidationUtil
::
IsBinaryMetricType
(
int32_t
metric_type
)
{
return
(
metric_type
==
static_cast
<
int32_t
>
(
engine
::
MetricType
::
HAMMING
))
||
(
metric_type
==
static_cast
<
int32_t
>
(
engine
::
MetricType
::
JACCARD
))
||
(
metric_type
==
static_cast
<
int32_t
>
(
engine
::
MetricType
::
TANIMOTO
));
}
Status
ValidationUtil
::
ValidateSearchTopk
(
int64_t
top_k
,
const
engine
::
meta
::
TableSchema
&
table_schema
)
{
if
(
top_k
<=
0
||
top_k
>
2048
)
{
...
...
core/src/utils/ValidationUtil.h
浏览文件 @
37fd7963
...
...
@@ -43,18 +43,12 @@ class ValidationUtil {
ValidateSearchParams
(
const
milvus
::
json
&
search_params
,
const
engine
::
meta
::
TableSchema
&
table_schema
,
int64_t
topk
);
static
bool
IsBinaryIndexType
(
int32_t
index_type
);
static
Status
ValidateTableIndexFileSize
(
int64_t
index_file_size
);
static
Status
ValidateTableIndexMetricType
(
int32_t
metric_type
);
static
bool
IsBinaryMetricType
(
int32_t
metric_type
);
static
Status
ValidateSearchTopk
(
int64_t
top_k
,
const
engine
::
meta
::
TableSchema
&
table_schema
);
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录