Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
3b68bf22
M
milvus
项目概览
milvus
/
milvus
11 个月 前同步成功
通知
260
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
前往新版Gitcode,体验更适合开发者的 AI 搜索 >>
提交
3b68bf22
编写于
1月 08, 2021
作者:
S
sunby
提交者:
yefu.chen
1月 08, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add pre-checks in create index task
Signed-off-by:
N
sunby
<
bingyi.sun@zilliz.com
>
上级
92b2df14
变更
7
隐藏空白更改
内联
并排
Showing
7 changed file
with
23 addition
and
58 deletion
+23
-58
internal/core/src/indexbuilder/IndexWrapper.cpp
internal/core/src/indexbuilder/IndexWrapper.cpp
+0
-18
internal/core/src/indexbuilder/IndexWrapper.h
internal/core/src/indexbuilder/IndexWrapper.h
+0
-3
internal/core/src/indexbuilder/utils.h
internal/core/src/indexbuilder/utils.h
+0
-15
internal/indexbuilder/index.go
internal/indexbuilder/index.go
+0
-7
internal/indexbuilder/task.go
internal/indexbuilder/task.go
+0
-7
internal/master/index_task.go
internal/master/index_task.go
+14
-5
internal/proxy/proxy_test.go
internal/proxy/proxy_test.go
+9
-3
未找到文件。
internal/core/src/indexbuilder/IndexWrapper.cpp
浏览文件 @
3b68bf22
...
...
@@ -34,9 +34,6 @@ IndexWrapper::IndexWrapper(const char* serialized_type_params, const char* seria
auto
mode
=
get_config_by_name
<
std
::
string
>
(
"index_mode"
);
auto
index_mode
=
mode
.
has_value
()
?
mode_map
[
mode
.
value
()]
:
knowhere
::
IndexMode
::
MODE_CPU
;
auto
index_type
=
get_index_type
();
auto
metric_type
=
get_metric_type
();
AssertInfo
(
!
is_unsupported
(
index_type
,
metric_type
),
index_type
+
" doesn't support metric: "
+
metric_type
);
index_
=
knowhere
::
VecIndexFactory
::
GetInstance
().
CreateVecIndex
(
get_index_type
(),
index_mode
);
Assert
(
index_
!=
nullptr
);
}
...
...
@@ -266,21 +263,6 @@ IndexWrapper::get_index_type() {
return
type
.
has_value
()
?
type
.
value
()
:
knowhere
::
IndexEnum
::
INDEX_FAISS_IVFPQ
;
}
std
::
string
IndexWrapper
::
get_metric_type
()
{
auto
type
=
get_config_by_name
<
std
::
string
>
(
knowhere
::
Metric
::
TYPE
);
if
(
type
.
has_value
())
{
return
type
.
value
();
}
else
{
auto
index_type
=
get_index_type
();
if
(
is_in_bin_list
(
index_type
))
{
return
knowhere
::
Metric
::
JACCARD
;
}
else
{
return
knowhere
::
Metric
::
L2
;
}
}
}
std
::
unique_ptr
<
IndexWrapper
::
QueryResult
>
IndexWrapper
::
Query
(
const
knowhere
::
DatasetPtr
&
dataset
)
{
return
std
::
move
(
QueryImpl
(
dataset
,
config_
));
...
...
internal/core/src/indexbuilder/IndexWrapper.h
浏览文件 @
3b68bf22
...
...
@@ -59,9 +59,6 @@ class IndexWrapper {
std
::
string
get_index_type
();
std
::
string
get_metric_type
();
template
<
typename
T
>
std
::
optional
<
T
>
get_config_by_name
(
std
::
string
name
);
...
...
internal/core/src/indexbuilder/utils.h
浏览文件 @
3b68bf22
...
...
@@ -14,7 +14,6 @@
#include <vector>
#include <string>
#include <algorithm>
#include <tuple>
#include "index/knowhere/knowhere/index/IndexType.h"
...
...
@@ -58,14 +57,6 @@ Need_BuildAll_list() {
return
ret
;
}
std
::
vector
<
std
::
tuple
<
std
::
string
,
std
::
string
>>
unsupported_index_combinations
()
{
static
std
::
vector
<
std
::
tuple
<
std
::
string
,
std
::
string
>>
ret
{
std
::
make_tuple
(
std
::
string
(
knowhere
::
IndexEnum
::
INDEX_FAISS_BIN_IVFFLAT
),
std
::
string
(
knowhere
::
Metric
::
L2
)),
};
return
ret
;
}
template
<
typename
T
>
bool
is_in_list
(
const
T
&
t
,
std
::
function
<
std
::
vector
<
T
>
()
>
list_func
)
{
...
...
@@ -93,11 +84,5 @@ is_in_need_id_list(const milvus::knowhere::IndexType& index_type) {
return
is_in_list
<
std
::
string
>
(
index_type
,
Need_ID_List
);
}
bool
is_unsupported
(
const
milvus
::
knowhere
::
IndexType
&
index_type
,
const
milvus
::
knowhere
::
MetricType
&
metric_type
)
{
return
is_in_list
<
std
::
tuple
<
std
::
string
,
std
::
string
>>
(
std
::
make_tuple
(
index_type
,
metric_type
),
unsupported_index_combinations
);
}
}
// namespace indexbuilder
}
// namespace milvus
internal/indexbuilder/index.go
浏览文件 @
3b68bf22
...
...
@@ -14,7 +14,6 @@ package indexbuilder
import
"C"
import
(
"errors"
"fmt"
"strconv"
"unsafe"
...
...
@@ -143,8 +142,6 @@ func (index *CIndex) Delete() error {
}
func
NewCIndex
(
typeParams
,
indexParams
map
[
string
]
string
)
(
Index
,
error
)
{
fmt
.
Println
(
"NNNNNNNNNNNNNNNNNNNNNNNNNNN typeParams: "
,
typeParams
)
fmt
.
Println
(
"NNNNNNNNNNNNNNNNNNNNNNNNNNN indexParams: "
,
indexParams
)
protoTypeParams
:=
&
indexcgopb
.
TypeParams
{
Params
:
make
([]
*
commonpb
.
KeyValuePair
,
0
),
}
...
...
@@ -171,14 +168,10 @@ func NewCIndex(typeParams, indexParams map[string]string) (Index, error) {
CIndex* res_index);
*/
var
indexPtr
C
.
CIndex
fmt
.
Println
(
"before create index ........................................"
)
status
:=
C
.
CreateIndex
(
typeParamsPointer
,
indexParamsPointer
,
&
indexPtr
)
fmt
.
Println
(
"after create index ........................................"
)
errorCode
:=
status
.
error_code
fmt
.
Println
(
"EEEEEEEEEEEEEEEEEEEEEEEEEE error code: "
,
errorCode
)
if
errorCode
!=
0
{
errorMsg
:=
C
.
GoString
(
status
.
error_msg
)
fmt
.
Println
(
"EEEEEEEEEEEEEEEEEEEEEEEEEE error msg: "
,
errorMsg
)
defer
C
.
free
(
unsafe
.
Pointer
(
status
.
error_msg
))
return
nil
,
errors
.
New
(
" failed, C runtime error detected, error code = "
+
strconv
.
Itoa
(
int
(
errorCode
))
+
", error msg = "
+
errorMsg
)
}
...
...
internal/indexbuilder/task.go
浏览文件 @
3b68bf22
...
...
@@ -2,7 +2,6 @@ package indexbuilder
import
(
"context"
"fmt"
"log"
"strconv"
"time"
...
...
@@ -172,12 +171,10 @@ func (it *IndexBuildTask) Execute() error {
indexParams
[
key
]
=
value
}
fmt
.
Println
(
"before NewCIndex .........................."
)
it
.
index
,
err
=
NewCIndex
(
typeParams
,
indexParams
)
if
err
!=
nil
{
return
err
}
fmt
.
Println
(
"after NewCIndex .........................."
)
getKeyByPathNaive
:=
func
(
path
string
)
string
{
// splitElements := strings.Split(path, "/")
...
...
@@ -226,7 +223,6 @@ func (it *IndexBuildTask) Execute() error {
for
_
,
value
:=
range
insertData
.
Data
{
// TODO: BinaryVectorFieldData
fmt
.
Println
(
"before build index .................................."
)
floatVectorFieldData
,
fOk
:=
value
.
(
*
storage
.
FloatVectorFieldData
)
if
fOk
{
err
=
it
.
index
.
BuildFloatVecIndexWithoutIds
(
floatVectorFieldData
.
Data
)
...
...
@@ -242,15 +238,12 @@ func (it *IndexBuildTask) Execute() error {
return
err
}
}
fmt
.
Println
(
"after build index .................................."
)
if
!
fOk
&&
!
bOk
{
return
errors
.
New
(
"we expect FloatVectorFieldData or BinaryVectorFieldData"
)
}
fmt
.
Println
(
"before serialize ............................................."
)
indexBlobs
,
err
:=
it
.
index
.
Serialize
()
fmt
.
Println
(
"after serialize ............................................."
)
if
err
!=
nil
{
return
err
}
...
...
internal/master/index_task.go
浏览文件 @
3b68bf22
...
...
@@ -24,11 +24,6 @@ func (task *createIndexTask) Ts() (Timestamp, error) {
}
func
(
task
*
createIndexTask
)
Execute
()
error
{
// modify schema
if
err
:=
task
.
mt
.
UpdateFieldIndexParams
(
task
.
req
.
CollectionName
,
task
.
req
.
FieldName
,
task
.
req
.
ExtraParams
);
err
!=
nil
{
return
err
}
// check if closed segment has the same index build history
collMeta
,
err
:=
task
.
mt
.
GetCollectionByName
(
task
.
req
.
CollectionName
)
if
err
!=
nil
{
return
err
...
...
@@ -44,6 +39,20 @@ func (task *createIndexTask) Execute() error {
return
fmt
.
Errorf
(
"can not find field name %s"
,
task
.
req
.
FieldName
)
}
// pre checks
isIndexable
,
err
:=
task
.
mt
.
IsIndexable
(
collMeta
.
ID
,
fieldID
)
if
err
!=
nil
{
return
err
}
if
!
isIndexable
{
return
fmt
.
Errorf
(
"field %s is not vector"
,
task
.
req
.
FieldName
)
}
// modify schema
if
err
:=
task
.
mt
.
UpdateFieldIndexParams
(
task
.
req
.
CollectionName
,
task
.
req
.
FieldName
,
task
.
req
.
ExtraParams
);
err
!=
nil
{
return
err
}
// check if closed segment has the same index build history
for
_
,
segID
:=
range
collMeta
.
SegmentIDs
{
segMeta
,
err
:=
task
.
mt
.
GetSegmentByID
(
segID
)
if
err
!=
nil
{
...
...
internal/proxy/proxy_test.go
浏览文件 @
3b68bf22
...
...
@@ -488,7 +488,9 @@ func TestProxy_CreateIndex(t *testing.T) {
go
func
(
group
*
sync
.
WaitGroup
)
{
defer
group
.
Done
()
createCollection
(
t
,
collName
)
createIndex
(
t
,
collName
,
fieldName
)
if
i
%
2
==
0
{
createIndex
(
t
,
collName
,
fieldName
)
}
dropCollection
(
t
,
collName
)
// dropIndex(t, collectionName, fieldName, indexName)
}(
&
wg
)
...
...
@@ -510,7 +512,9 @@ func TestProxy_DescribeIndex(t *testing.T) {
go
func
(
group
*
sync
.
WaitGroup
)
{
defer
group
.
Done
()
createCollection
(
t
,
collName
)
createIndex
(
t
,
collName
,
fieldName
)
if
i
%
2
==
0
{
createIndex
(
t
,
collName
,
fieldName
)
}
req
:=
&
servicepb
.
DescribeIndexRequest
{
CollectionName
:
collName
,
FieldName
:
fieldName
,
...
...
@@ -539,7 +543,9 @@ func TestProxy_DescribeIndexProgress(t *testing.T) {
go
func
(
group
*
sync
.
WaitGroup
)
{
defer
group
.
Done
()
createCollection
(
t
,
collName
)
createIndex
(
t
,
collName
,
fieldName
)
if
i
%
2
==
0
{
createIndex
(
t
,
collName
,
fieldName
)
}
req
:=
&
servicepb
.
DescribeIndexProgressRequest
{
CollectionName
:
collName
,
FieldName
:
fieldName
,
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录