Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
bb8da156
M
milvus
项目概览
milvus
/
milvus
大约 1 年 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
bb8da156
编写于
12月 29, 2020
作者:
B
bigsheeper
提交者:
yefu.chen
12月 29, 2020
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Return status and add try catch exception in index builder
Signed-off-by:
N
bigsheeper
<
yihao.dai@zilliz.com
>
上级
b1b87474
变更
10
隐藏空白更改
内联
并排
Showing
10 changed file
with
156 addition
and
49 deletion
+156
-49
internal/core/CMakeLists.txt
internal/core/CMakeLists.txt
+6
-0
internal/core/src/common/status_c.h
internal/core/src/common/status_c.h
+30
-0
internal/core/src/indexbuilder/index_c.cpp
internal/core/src/indexbuilder/index_c.cpp
+66
-23
internal/core/src/indexbuilder/index_c.h
internal/core/src/indexbuilder/index_c.h
+7
-6
internal/core/src/segcore/collection_c.h
internal/core/src/segcore/collection_c.h
+0
-10
internal/core/src/segcore/load_index_c.h
internal/core/src/segcore/load_index_c.h
+1
-0
internal/core/src/segcore/plan_c.h
internal/core/src/segcore/plan_c.h
+1
-0
internal/core/src/segcore/reduce_c.h
internal/core/src/segcore/reduce_c.h
+1
-0
internal/core/src/segcore/segment_c.h
internal/core/src/segcore/segment_c.h
+1
-0
internal/indexbuilder/index.go
internal/indexbuilder/index.go
+43
-10
未找到文件。
internal/core/CMakeLists.txt
浏览文件 @
bb8da156
...
...
@@ -214,3 +214,9 @@ install(
install
(
FILES
${
CMAKE_BINARY_DIR
}
/src/indexbuilder/libmilvus_indexbuilder.so
DESTINATION lib
)
install
(
DIRECTORY
${
CMAKE_CURRENT_SOURCE_DIR
}
/src/common/
DESTINATION include/common/
FILES_MATCHING PATTERN
"*_c.h"
)
internal/core/src/common/status_c.h
0 → 100644
浏览文件 @
bb8da156
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#pragma once
#ifdef __cplusplus
extern
"C"
{
#endif
enum
ErrorCode
{
Success
=
0
,
UnexpectedException
=
1
,
};
typedef
struct
CStatus
{
int
error_code
;
const
char
*
error_msg
;
}
CStatus
;
#ifdef __cplusplus
}
#endif
internal/core/src/indexbuilder/index_c.cpp
浏览文件 @
bb8da156
...
...
@@ -26,11 +26,27 @@ class CGODebugUtils {
}
};
CIndex
CreateIndex
(
const
char
*
serialized_type_params
,
const
char
*
serialized_index_params
)
{
auto
index
=
std
::
make_unique
<
milvus
::
indexbuilder
::
IndexWrapper
>
(
serialized_type_params
,
serialized_index_params
);
return
index
.
release
();
CStatus
CreateIndex
(
const
char
*
serialized_type_params
,
const
char
*
serialized_index_params
,
CIndex
*
res_index
)
{
auto
status
=
CStatus
();
try
{
// std::cout << "strlen(serialized_type_params): " << CGODebugUtils::Strlen(serialized_type_params,
// type_params_size)
// << std::endl;
// std::cout << "type_params_size: " << type_params_size << std::endl;
// std::cout << "strlen(serialized_index_params): "
// << CGODebugUtils::Strlen(serialized_index_params, index_params_size) << std::endl;
// std::cout << "index_params_size: " << index_params_size << std::endl;
auto
index
=
std
::
make_unique
<
milvus
::
indexbuilder
::
IndexWrapper
>
(
serialized_type_params
,
serialized_index_params
);
*
res_index
=
index
.
release
();
status
.
error_code
=
Success
;
status
.
error_msg
=
""
;
}
catch
(
std
::
runtime_error
&
e
)
{
status
.
error_code
=
UnexpectedException
;
status
.
error_msg
=
strdup
(
e
.
what
());
}
return
status
;
}
void
...
...
@@ -39,30 +55,57 @@ DeleteIndex(CIndex index) {
delete
cIndex
;
}
void
CStatus
BuildFloatVecIndexWithoutIds
(
CIndex
index
,
int64_t
float_value_num
,
const
float
*
vectors
)
{
auto
cIndex
=
(
milvus
::
indexbuilder
::
IndexWrapper
*
)
index
;
auto
dim
=
cIndex
->
dim
();
auto
row_nums
=
float_value_num
/
dim
;
auto
ds
=
milvus
::
knowhere
::
GenDataset
(
row_nums
,
dim
,
vectors
);
cIndex
->
BuildWithoutIds
(
ds
);
auto
status
=
CStatus
();
try
{
auto
cIndex
=
(
milvus
::
indexbuilder
::
IndexWrapper
*
)
index
;
auto
dim
=
cIndex
->
dim
();
auto
row_nums
=
float_value_num
/
dim
;
auto
ds
=
milvus
::
knowhere
::
GenDataset
(
row_nums
,
dim
,
vectors
);
cIndex
->
BuildWithoutIds
(
ds
);
status
.
error_code
=
Success
;
status
.
error_msg
=
""
;
}
catch
(
std
::
runtime_error
&
e
)
{
status
.
error_code
=
UnexpectedException
;
status
.
error_msg
=
strdup
(
e
.
what
());
}
return
status
;
}
void
CStatus
BuildBinaryVecIndexWithoutIds
(
CIndex
index
,
int64_t
data_size
,
const
uint8_t
*
vectors
)
{
auto
cIndex
=
(
milvus
::
indexbuilder
::
IndexWrapper
*
)
index
;
auto
dim
=
cIndex
->
dim
();
auto
row_nums
=
(
data_size
*
8
)
/
dim
;
auto
ds
=
milvus
::
knowhere
::
GenDataset
(
row_nums
,
dim
,
vectors
);
cIndex
->
BuildWithoutIds
(
ds
);
auto
status
=
CStatus
();
try
{
auto
cIndex
=
(
milvus
::
indexbuilder
::
IndexWrapper
*
)
index
;
auto
dim
=
cIndex
->
dim
();
auto
row_nums
=
(
data_size
*
8
)
/
dim
;
auto
ds
=
milvus
::
knowhere
::
GenDataset
(
row_nums
,
dim
,
vectors
);
cIndex
->
BuildWithoutIds
(
ds
);
status
.
error_code
=
Success
;
status
.
error_msg
=
""
;
}
catch
(
std
::
runtime_error
&
e
)
{
status
.
error_code
=
UnexpectedException
;
status
.
error_msg
=
strdup
(
e
.
what
());
}
return
status
;
}
char
*
SerializeToSlicedBuffer
(
CIndex
index
,
int32_t
*
buffer_size
)
{
auto
cIndex
=
(
milvus
::
indexbuilder
::
IndexWrapper
*
)
index
;
auto
binary
=
cIndex
->
Serialize
();
*
buffer_size
=
binary
.
size
;
return
binary
.
data
;
CStatus
SerializeToSlicedBuffer
(
CIndex
index
,
int32_t
*
buffer_size
,
char
**
res_buffer
)
{
auto
status
=
CStatus
();
try
{
auto
cIndex
=
(
milvus
::
indexbuilder
::
IndexWrapper
*
)
index
;
auto
binary
=
cIndex
->
Serialize
();
*
buffer_size
=
binary
.
size
;
*
res_buffer
=
binary
.
data
;
status
.
error_code
=
Success
;
status
.
error_msg
=
""
;
}
catch
(
std
::
runtime_error
&
e
)
{
status
.
error_code
=
UnexpectedException
;
status
.
error_msg
=
strdup
(
e
.
what
());
}
return
status
;
}
void
...
...
internal/core/src/indexbuilder/index_c.h
浏览文件 @
bb8da156
...
...
@@ -28,26 +28,27 @@ extern "C" {
#include <stdint.h>
#include "segcore/collection_c.h"
#include "common/status_c.h"
typedef
void
*
CIndex
;
// TODO: how could we pass map between go and c++ more efficiently?
// Solution: using protobuf instead of json, this way significantly increase programming efficiency
C
Index
CreateIndex
(
const
char
*
serialized_type_params
,
const
char
*
serialized_index_params
);
C
Status
CreateIndex
(
const
char
*
serialized_type_params
,
const
char
*
serialized_index_params
,
CIndex
*
res_index
);
void
DeleteIndex
(
CIndex
index
);
void
CStatus
BuildFloatVecIndexWithoutIds
(
CIndex
index
,
int64_t
float_value_num
,
const
float
*
vectors
);
void
CStatus
BuildBinaryVecIndexWithoutIds
(
CIndex
index
,
int64_t
data_size
,
const
uint8_t
*
vectors
);
char
*
SerializeToSlicedBuffer
(
CIndex
index
,
int32_t
*
buffer_size
);
CStatus
SerializeToSlicedBuffer
(
CIndex
index
,
int32_t
*
buffer_size
,
char
**
res_buffer
);
void
LoadFromSlicedBuffer
(
CIndex
index
,
const
char
*
serialized_sliced_blob_buffer
,
int32_t
size
);
...
...
internal/core/src/segcore/collection_c.h
浏览文件 @
bb8da156
...
...
@@ -15,16 +15,6 @@
extern
"C"
{
#endif
enum
ErrorCode
{
Success
=
0
,
UnexpectedException
=
1
,
};
typedef
struct
CStatus
{
int
error_code
;
const
char
*
error_msg
;
}
CStatus
;
typedef
void
*
CCollection
;
CCollection
...
...
internal/core/src/segcore/load_index_c.h
浏览文件 @
bb8da156
...
...
@@ -18,6 +18,7 @@ extern "C" {
#include <stdint.h>
#include "segcore/collection_c.h"
#include "common/status_c.h"
typedef
void
*
CLoadIndexInfo
;
typedef
void
*
CBinarySet
;
...
...
internal/core/src/segcore/plan_c.h
浏览文件 @
bb8da156
...
...
@@ -16,6 +16,7 @@ extern "C" {
#include <stdbool.h>
#include <stdint.h>
#include "segcore/collection_c.h"
#include "common/status_c.h"
typedef
void
*
CPlan
;
typedef
void
*
CPlaceholderGroup
;
...
...
internal/core/src/segcore/reduce_c.h
浏览文件 @
bb8da156
...
...
@@ -16,6 +16,7 @@ extern "C" {
#include <stdbool.h>
#include <stdint.h>
#include "segcore/segment_c.h"
#include "common/status_c.h"
typedef
void
*
CMarshaledHits
;
...
...
internal/core/src/segcore/segment_c.h
浏览文件 @
bb8da156
...
...
@@ -19,6 +19,7 @@ extern "C" {
#include "segcore/plan_c.h"
#include "segcore/load_index_c.h"
#include "common/status_c.h"
typedef
void
*
CSegmentBase
;
typedef
void
*
CQueryResult
;
...
...
internal/indexbuilder/index.go
浏览文件 @
bb8da156
...
...
@@ -13,6 +13,8 @@ package indexbuilder
*/
import
"C"
import
(
"errors"
"strconv"
"unsafe"
"github.com/golang/protobuf/proto"
...
...
@@ -38,11 +40,20 @@ type CIndex struct {
func
(
index
*
CIndex
)
Serialize
()
([]
*
Blob
,
error
)
{
/*
char*
SerializeToSlicedBuffer(CIndex index, int32_t* buffer_size);
CStatus
SerializeToSlicedBuffer(CIndex index, int32_t* buffer_size
, char** res_buffer
);
*/
var
cDumpedSlicedBuffer
*
C
.
char
var
bufferSize
int32
var
cDumpedSlicedBuffer
*
C
.
char
=
C
.
SerializeToSlicedBuffer
(
index
.
indexPtr
,
(
*
C
.
int32_t
)(
unsafe
.
Pointer
(
&
bufferSize
)))
status
:=
C
.
SerializeToSlicedBuffer
(
index
.
indexPtr
,
(
*
C
.
int32_t
)(
unsafe
.
Pointer
(
&
bufferSize
)),
&
cDumpedSlicedBuffer
)
errorCode
:=
status
.
error_code
if
errorCode
!=
0
{
errorMsg
:=
C
.
GoString
(
status
.
error_msg
)
defer
C
.
free
(
unsafe
.
Pointer
(
status
.
error_msg
))
return
nil
,
errors
.
New
(
"SerializeToSlicedBuffer failed, C runtime error detected, error code = "
+
strconv
.
Itoa
(
int
(
errorCode
))
+
", error msg = "
+
errorMsg
)
}
defer
C
.
free
(
unsafe
.
Pointer
(
cDumpedSlicedBuffer
))
dumpedSlicedBuffer
:=
C
.
GoBytes
(
unsafe
.
Pointer
(
cDumpedSlicedBuffer
),
(
C
.
int32_t
)(
bufferSize
))
...
...
@@ -81,19 +92,31 @@ func (index *CIndex) Load(blobs []*Blob) error {
func
(
index
*
CIndex
)
BuildFloatVecIndexWithoutIds
(
vectors
[]
float32
)
error
{
/*
void
CStatus
BuildFloatVecIndexWithoutIds(CIndex index, int64_t float_value_num, const float* vectors);
*/
C
.
BuildFloatVecIndexWithoutIds
(
index
.
indexPtr
,
(
C
.
int64_t
)(
len
(
vectors
)),
(
*
C
.
float
)(
&
vectors
[
0
]))
status
:=
C
.
BuildFloatVecIndexWithoutIds
(
index
.
indexPtr
,
(
C
.
int64_t
)(
len
(
vectors
)),
(
*
C
.
float
)(
&
vectors
[
0
]))
errorCode
:=
status
.
error_code
if
errorCode
!=
0
{
errorMsg
:=
C
.
GoString
(
status
.
error_msg
)
defer
C
.
free
(
unsafe
.
Pointer
(
status
.
error_msg
))
return
errors
.
New
(
"BuildFloatVecIndexWithoutIds failed, C runtime error detected, error code = "
+
strconv
.
Itoa
(
int
(
errorCode
))
+
", error msg = "
+
errorMsg
)
}
return
nil
}
func
(
index
*
CIndex
)
BuildBinaryVecIndexWithoutIds
(
vectors
[]
byte
)
error
{
/*
void
CStatus
BuildBinaryVecIndexWithoutIds(CIndex index, int64_t data_size, const uint8_t* vectors);
*/
C
.
BuildBinaryVecIndexWithoutIds
(
index
.
indexPtr
,
(
C
.
int64_t
)(
len
(
vectors
)),
(
*
C
.
uint8_t
)(
&
vectors
[
0
]))
status
:=
C
.
BuildBinaryVecIndexWithoutIds
(
index
.
indexPtr
,
(
C
.
int64_t
)(
len
(
vectors
)),
(
*
C
.
uint8_t
)(
&
vectors
[
0
]))
errorCode
:=
status
.
error_code
if
errorCode
!=
0
{
errorMsg
:=
C
.
GoString
(
status
.
error_msg
)
defer
C
.
free
(
unsafe
.
Pointer
(
status
.
error_msg
))
return
errors
.
New
(
" failed, C runtime error detected, error code = "
+
strconv
.
Itoa
(
int
(
errorCode
))
+
", error msg = "
+
errorMsg
)
}
return
nil
}
...
...
@@ -127,11 +150,21 @@ func NewCIndex(typeParams, indexParams map[string]string) (Index, error) {
indexParamsPointer
:=
C
.
CString
(
indexParamsStr
)
/*
C
Index
C
Status
CreateIndex(const char* serialized_type_params,
const char* serialized_index_params);
const char* serialized_index_params,
CIndex* res_index);
*/
var
indexPtr
C
.
CIndex
status
:=
C
.
CreateIndex
(
typeParamsPointer
,
indexParamsPointer
,
&
indexPtr
)
errorCode
:=
status
.
error_code
if
errorCode
!=
0
{
errorMsg
:=
C
.
GoString
(
status
.
error_msg
)
defer
C
.
free
(
unsafe
.
Pointer
(
status
.
error_msg
))
return
nil
,
errors
.
New
(
" failed, C runtime error detected, error code = "
+
strconv
.
Itoa
(
int
(
errorCode
))
+
", error msg = "
+
errorMsg
)
}
return
&
CIndex
{
indexPtr
:
C
.
CreateIndex
(
typeParamsPointer
,
indexParamsPointer
)
,
indexPtr
:
indexPtr
,
},
nil
}
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录