Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
c5149c7e
M
milvus
项目概览
milvus
/
milvus
大约 1 年 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
c5149c7e
编写于
5月 13, 2022
作者:
J
Jiquan Long
提交者:
GitHub
5月 13, 2022
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Replace text codec of pb with binary codec (#16955)
Signed-off-by:
N
dragondriver
<
jiquan.long@zilliz.com
>
上级
63385b7c
变更
5
展开全部
隐藏空白更改
内联
并排
Showing
5 changed file
with
147 addition
and
209 deletion
+147
-209
internal/core/src/common/type_c.h
internal/core/src/common/type_c.h
+4
-2
internal/core/src/segcore/segment_c.cpp
internal/core/src/segcore/segment_c.cpp
+14
-17
internal/core/src/segcore/segment_c.h
internal/core/src/segcore/segment_c.h
+8
-2
internal/core/unittest/test_c_api.cpp
internal/core/unittest/test_c_api.cpp
+96
-149
internal/querynode/segment.go
internal/querynode/segment.go
+25
-39
未找到文件。
internal/core/src/common/type_c.h
浏览文件 @
c5149c7e
...
@@ -70,13 +70,15 @@ typedef struct CProto {
...
@@ -70,13 +70,15 @@ typedef struct CProto {
typedef
struct
CLoadFieldDataInfo
{
typedef
struct
CLoadFieldDataInfo
{
int64_t
field_id
;
int64_t
field_id
;
const
char
*
blob
;
const
uint8_t
*
blob
;
uint64_t
blob_size
;
int64_t
row_count
;
int64_t
row_count
;
}
CLoadFieldDataInfo
;
}
CLoadFieldDataInfo
;
typedef
struct
CLoadDeletedRecordInfo
{
typedef
struct
CLoadDeletedRecordInfo
{
void
*
timestamps
;
void
*
timestamps
;
const
char
*
primary_keys
;
const
uint8_t
*
primary_keys
;
const
uint64_t
primary_keys_size
;
int64_t
row_count
;
int64_t
row_count
;
}
CLoadDeletedRecordInfo
;
}
CLoadDeletedRecordInfo
;
...
...
internal/core/src/segcore/segment_c.cpp
浏览文件 @
c5149c7e
...
@@ -134,14 +134,13 @@ Insert(CSegmentInterface c_segment,
...
@@ -134,14 +134,13 @@ Insert(CSegmentInterface c_segment,
int64_t
size
,
int64_t
size
,
const
int64_t
*
row_ids
,
const
int64_t
*
row_ids
,
const
uint64_t
*
timestamps
,
const
uint64_t
*
timestamps
,
const
char
*
data_info
)
{
const
uint8_t
*
data_info
,
const
uint64_t
data_info_len
)
{
try
{
try
{
auto
segment
=
(
milvus
::
segcore
::
SegmentGrowing
*
)
c_segment
;
auto
segment
=
(
milvus
::
segcore
::
SegmentGrowing
*
)
c_segment
;
auto
proto
=
std
::
string
(
data_info
);
Assert
(
!
proto
.
empty
());
auto
insert_data
=
std
::
make_unique
<
milvus
::
InsertData
>
();
auto
insert_data
=
std
::
make_unique
<
milvus
::
InsertData
>
();
auto
suc
=
google
::
protobuf
::
TextFormat
::
ParseFromString
(
proto
,
insert_data
.
get
()
);
auto
suc
=
insert_data
->
ParseFromArray
(
data_info
,
data_info_len
);
AssertInfo
(
suc
,
"
unmarshal field data string failed
"
);
AssertInfo
(
suc
,
"
failed to parse insert data from records
"
);
segment
->
Insert
(
reserved_offset
,
size
,
row_ids
,
timestamps
,
insert_data
.
get
());
segment
->
Insert
(
reserved_offset
,
size
,
row_ids
,
timestamps
,
insert_data
.
get
());
return
milvus
::
SuccessCStatus
();
return
milvus
::
SuccessCStatus
();
...
@@ -162,14 +161,16 @@ PreInsert(CSegmentInterface c_segment, int64_t size, int64_t* offset) {
...
@@ -162,14 +161,16 @@ PreInsert(CSegmentInterface c_segment, int64_t size, int64_t* offset) {
}
}
CStatus
CStatus
Delete
(
Delete
(
CSegmentInterface
c_segment
,
CSegmentInterface
c_segment
,
int64_t
reserved_offset
,
int64_t
size
,
const
char
*
ids
,
const
uint64_t
*
timestamps
)
{
int64_t
reserved_offset
,
int64_t
size
,
const
uint8_t
*
ids
,
const
uint64_t
ids_size
,
const
uint64_t
*
timestamps
)
{
auto
segment
=
(
milvus
::
segcore
::
SegmentInterface
*
)
c_segment
;
auto
segment
=
(
milvus
::
segcore
::
SegmentInterface
*
)
c_segment
;
auto
proto
=
std
::
string
(
ids
);
Assert
(
!
proto
.
empty
());
auto
pks
=
std
::
make_unique
<
milvus
::
proto
::
schema
::
IDs
>
();
auto
pks
=
std
::
make_unique
<
milvus
::
proto
::
schema
::
IDs
>
();
auto
suc
=
google
::
protobuf
::
TextFormat
::
ParseFromString
(
proto
,
pks
.
get
()
);
auto
suc
=
pks
->
ParseFromArray
(
ids
,
ids_size
);
AssertInfo
(
suc
,
"
unmarshal field data string failed
"
);
AssertInfo
(
suc
,
"
failed to parse pks from ids
"
);
try
{
try
{
auto
res
=
segment
->
Delete
(
reserved_offset
,
size
,
pks
.
get
(),
timestamps
);
auto
res
=
segment
->
Delete
(
reserved_offset
,
size
,
pks
.
get
(),
timestamps
);
return
milvus
::
SuccessCStatus
();
return
milvus
::
SuccessCStatus
();
...
@@ -192,10 +193,8 @@ LoadFieldData(CSegmentInterface c_segment, CLoadFieldDataInfo load_field_data_in
...
@@ -192,10 +193,8 @@ LoadFieldData(CSegmentInterface c_segment, CLoadFieldDataInfo load_field_data_in
auto
segment_interface
=
reinterpret_cast
<
milvus
::
segcore
::
SegmentInterface
*>
(
c_segment
);
auto
segment_interface
=
reinterpret_cast
<
milvus
::
segcore
::
SegmentInterface
*>
(
c_segment
);
auto
segment
=
dynamic_cast
<
milvus
::
segcore
::
SegmentSealed
*>
(
segment_interface
);
auto
segment
=
dynamic_cast
<
milvus
::
segcore
::
SegmentSealed
*>
(
segment_interface
);
AssertInfo
(
segment
!=
nullptr
,
"segment conversion failed"
);
AssertInfo
(
segment
!=
nullptr
,
"segment conversion failed"
);
auto
proto
=
std
::
string
(
load_field_data_info
.
blob
);
Assert
(
!
proto
.
empty
());
auto
field_data
=
std
::
make_unique
<
milvus
::
DataArray
>
();
auto
field_data
=
std
::
make_unique
<
milvus
::
DataArray
>
();
auto
suc
=
google
::
protobuf
::
TextFormat
::
ParseFromString
(
proto
,
field_data
.
get
()
);
auto
suc
=
field_data
->
ParseFromArray
(
load_field_data_info
.
blob
,
load_field_data_info
.
blob_size
);
AssertInfo
(
suc
,
"unmarshal field data string failed"
);
AssertInfo
(
suc
,
"unmarshal field data string failed"
);
auto
load_info
=
auto
load_info
=
LoadFieldDataInfo
{
load_field_data_info
.
field_id
,
field_data
.
get
(),
load_field_data_info
.
row_count
};
LoadFieldDataInfo
{
load_field_data_info
.
field_id
,
field_data
.
get
(),
load_field_data_info
.
row_count
};
...
@@ -211,10 +210,8 @@ LoadDeletedRecord(CSegmentInterface c_segment, CLoadDeletedRecordInfo deleted_re
...
@@ -211,10 +210,8 @@ LoadDeletedRecord(CSegmentInterface c_segment, CLoadDeletedRecordInfo deleted_re
try
{
try
{
auto
segment_interface
=
reinterpret_cast
<
milvus
::
segcore
::
SegmentInterface
*>
(
c_segment
);
auto
segment_interface
=
reinterpret_cast
<
milvus
::
segcore
::
SegmentInterface
*>
(
c_segment
);
AssertInfo
(
segment_interface
!=
nullptr
,
"segment conversion failed"
);
AssertInfo
(
segment_interface
!=
nullptr
,
"segment conversion failed"
);
auto
proto
=
std
::
string
(
deleted_record_info
.
primary_keys
);
Assert
(
!
proto
.
empty
());
auto
pks
=
std
::
make_unique
<
milvus
::
proto
::
schema
::
IDs
>
();
auto
pks
=
std
::
make_unique
<
milvus
::
proto
::
schema
::
IDs
>
();
auto
suc
=
google
::
protobuf
::
TextFormat
::
ParseFromString
(
proto
,
pks
.
get
()
);
auto
suc
=
pks
->
ParseFromArray
(
deleted_record_info
.
primary_keys
,
deleted_record_info
.
primary_keys_size
);
AssertInfo
(
suc
,
"unmarshal field data string failed"
);
AssertInfo
(
suc
,
"unmarshal field data string failed"
);
auto
load_info
=
auto
load_info
=
LoadDeletedRecordInfo
{
deleted_record_info
.
timestamps
,
pks
.
get
(),
deleted_record_info
.
row_count
};
LoadDeletedRecordInfo
{
deleted_record_info
.
timestamps
,
pks
.
get
(),
deleted_record_info
.
row_count
};
...
...
internal/core/src/segcore/segment_c.h
浏览文件 @
c5149c7e
...
@@ -67,7 +67,8 @@ Insert(CSegmentInterface c_segment,
...
@@ -67,7 +67,8 @@ Insert(CSegmentInterface c_segment,
int64_t
size
,
int64_t
size
,
const
int64_t
*
row_ids
,
const
int64_t
*
row_ids
,
const
uint64_t
*
timestamps
,
const
uint64_t
*
timestamps
,
const
char
*
data_info
);
const
uint8_t
*
data_info
,
const
uint64_t
data_info_len
);
CStatus
CStatus
PreInsert
(
CSegmentInterface
c_segment
,
int64_t
size
,
int64_t
*
offset
);
PreInsert
(
CSegmentInterface
c_segment
,
int64_t
size
,
int64_t
*
offset
);
...
@@ -90,7 +91,12 @@ DropSealedSegmentIndex(CSegmentInterface c_segment, int64_t field_id);
...
@@ -90,7 +91,12 @@ DropSealedSegmentIndex(CSegmentInterface c_segment, int64_t field_id);
////////////////////////////// interfaces for SegmentInterface //////////////////////////////
////////////////////////////// interfaces for SegmentInterface //////////////////////////////
CStatus
CStatus
Delete
(
CSegmentInterface
c_segment
,
int64_t
reserved_offset
,
int64_t
size
,
const
char
*
ids
,
const
uint64_t
*
timestamps
);
Delete
(
CSegmentInterface
c_segment
,
int64_t
reserved_offset
,
int64_t
size
,
const
uint8_t
*
ids
,
const
uint64_t
ids_size
,
const
uint64_t
*
timestamps
);
int64_t
int64_t
PreDelete
(
CSegmentInterface
c_segment
,
int64_t
size
);
PreDelete
(
CSegmentInterface
c_segment
,
int64_t
size
);
...
...
internal/core/unittest/test_c_api.cpp
浏览文件 @
c5149c7e
此差异已折叠。
点击以展开。
internal/querynode/segment.go
浏览文件 @
c5149c7e
...
@@ -609,17 +609,6 @@ func (s *Segment) segmentPreDelete(numOfRecords int) int64 {
...
@@ -609,17 +609,6 @@ func (s *Segment) segmentPreDelete(numOfRecords int) int64 {
}
}
func
(
s
*
Segment
)
segmentInsert
(
offset
int64
,
entityIDs
[]
UniqueID
,
timestamps
[]
Timestamp
,
record
*
segcorepb
.
InsertRecord
)
error
{
func
(
s
*
Segment
)
segmentInsert
(
offset
int64
,
entityIDs
[]
UniqueID
,
timestamps
[]
Timestamp
,
record
*
segcorepb
.
InsertRecord
)
error
{
/*
CStatus
Insert(CSegmentInterface c_segment,
long int reserved_offset,
signed long int size,
const long* primary_keys,
const unsigned long* timestamps,
void* raw_data,
int sizeof_per_row,
signed long int count);
*/
s
.
segPtrMu
.
RLock
()
s
.
segPtrMu
.
RLock
()
defer
s
.
segPtrMu
.
RUnlock
()
// thread safe guaranteed by segCore, use RLock
defer
s
.
segPtrMu
.
RUnlock
()
// thread safe guaranteed by segCore, use RLock
if
s
.
segmentType
!=
segmentTypeGrowing
{
if
s
.
segmentType
!=
segmentTypeGrowing
{
...
@@ -630,10 +619,10 @@ func (s *Segment) segmentInsert(offset int64, entityIDs []UniqueID, timestamps [
...
@@ -630,10 +619,10 @@ func (s *Segment) segmentInsert(offset int64, entityIDs []UniqueID, timestamps [
return
errors
.
New
(
"null seg core pointer"
)
return
errors
.
New
(
"null seg core pointer"
)
}
}
insertRecordBlob
:=
proto
.
MarshalTextString
(
record
)
insertRecordBlob
,
err
:=
proto
.
Marshal
(
record
)
if
err
!=
nil
{
cInsertRecordBlob
:=
C
.
CString
(
insertRecordBlob
)
return
fmt
.
Errorf
(
"failed to marshal insert record: %s"
,
err
)
defer
C
.
free
(
unsafe
.
Pointer
(
cInsertRecordBlob
))
}
var
numOfRow
=
len
(
entityIDs
)
var
numOfRow
=
len
(
entityIDs
)
var
cOffset
=
C
.
int64_t
(
offset
)
var
cOffset
=
C
.
int64_t
(
offset
)
...
@@ -646,7 +635,8 @@ func (s *Segment) segmentInsert(offset int64, entityIDs []UniqueID, timestamps [
...
@@ -646,7 +635,8 @@ func (s *Segment) segmentInsert(offset int64, entityIDs []UniqueID, timestamps [
cNumOfRows
,
cNumOfRows
,
cEntityIdsPtr
,
cEntityIdsPtr
,
cTimestampsPtr
,
cTimestampsPtr
,
cInsertRecordBlob
)
(
*
C
.
uint8_t
)(
unsafe
.
Pointer
(
&
insertRecordBlob
[
0
])),
(
C
.
uint64_t
)(
len
(
insertRecordBlob
)))
if
err
:=
HandleCStatus
(
&
status
,
"Insert failed"
);
err
!=
nil
{
if
err
:=
HandleCStatus
(
&
status
,
"Insert failed"
);
err
!=
nil
{
return
err
return
err
}
}
...
@@ -709,11 +699,12 @@ func (s *Segment) segmentDelete(offset int64, entityIDs []primaryKey, timestamps
...
@@ -709,11 +699,12 @@ func (s *Segment) segmentDelete(offset int64, entityIDs []primaryKey, timestamps
return
fmt
.
Errorf
(
"invalid data type of primary keys"
)
return
fmt
.
Errorf
(
"invalid data type of primary keys"
)
}
}
dataBlob
:=
proto
.
MarshalTextString
(
ids
)
dataBlob
,
err
:=
proto
.
Marshal
(
ids
)
cDataBlob
:=
C
.
CString
(
dataBlob
)
if
err
!=
nil
{
defer
C
.
free
(
unsafe
.
Pointer
(
cDataBlob
))
return
fmt
.
Errorf
(
"failed to marshal ids: %s"
,
err
)
}
status
:=
C
.
Delete
(
s
.
segmentPtr
,
cOffset
,
cSize
,
cDataBlob
,
cTimestampsPtr
)
status
:=
C
.
Delete
(
s
.
segmentPtr
,
cOffset
,
cSize
,
(
*
C
.
uint8_t
)(
unsafe
.
Pointer
(
&
dataBlob
[
0
])),
(
C
.
uint64_t
)(
len
(
dataBlob
))
,
cTimestampsPtr
)
if
err
:=
HandleCStatus
(
&
status
,
"Delete failed"
);
err
!=
nil
{
if
err
:=
HandleCStatus
(
&
status
,
"Delete failed"
);
err
!=
nil
{
return
err
return
err
}
}
...
@@ -737,21 +728,15 @@ func (s *Segment) segmentLoadFieldData(fieldID int64, rowCount int64, data *sche
...
@@ -737,21 +728,15 @@ func (s *Segment) segmentLoadFieldData(fieldID int64, rowCount int64, data *sche
return
errors
.
New
(
errMsg
)
return
errors
.
New
(
errMsg
)
}
}
dataBlob
:=
proto
.
MarshalTextString
(
data
)
dataBlob
,
err
:=
proto
.
Marshal
(
data
)
if
err
!=
nil
{
cDataBlob
:=
C
.
CString
(
dataBlob
)
return
err
defer
C
.
free
(
unsafe
.
Pointer
(
cDataBlob
))
}
/*
typedef struct CLoadFieldDataInfo {
int64_t field_id;
void* blob;
int64_t row_count;
} CLoadFieldDataInfo;
*/
loadInfo
:=
C
.
CLoadFieldDataInfo
{
loadInfo
:=
C
.
CLoadFieldDataInfo
{
field_id
:
C
.
int64_t
(
fieldID
),
field_id
:
C
.
int64_t
(
fieldID
),
blob
:
cDataBlob
,
blob
:
(
*
C
.
uint8_t
)(
unsafe
.
Pointer
(
&
dataBlob
[
0
])),
blob_size
:
C
.
uint64_t
(
len
(
dataBlob
)),
row_count
:
C
.
int64_t
(
rowCount
),
row_count
:
C
.
int64_t
(
rowCount
),
}
}
...
@@ -805,15 +790,16 @@ func (s *Segment) segmentLoadDeletedRecord(primaryKeys []primaryKey, timestamps
...
@@ -805,15 +790,16 @@ func (s *Segment) segmentLoadDeletedRecord(primaryKeys []primaryKey, timestamps
return
fmt
.
Errorf
(
"invalid data type of primary keys"
)
return
fmt
.
Errorf
(
"invalid data type of primary keys"
)
}
}
idsBlob
:=
proto
.
MarshalTextString
(
ids
)
idsBlob
,
err
:=
proto
.
Marshal
(
ids
)
if
err
!=
nil
{
cIdsBlob
:=
C
.
CString
(
idsBlob
)
return
err
defer
C
.
free
(
unsafe
.
Pointer
(
cIdsBlob
))
}
loadInfo
:=
C
.
CLoadDeletedRecordInfo
{
loadInfo
:=
C
.
CLoadDeletedRecordInfo
{
timestamps
:
unsafe
.
Pointer
(
&
timestamps
[
0
]),
timestamps
:
unsafe
.
Pointer
(
&
timestamps
[
0
]),
primary_keys
:
cIdsBlob
,
primary_keys
:
(
*
C
.
uint8_t
)(
unsafe
.
Pointer
(
&
idsBlob
[
0
])),
row_count
:
C
.
int64_t
(
rowCount
),
primary_keys_size
:
C
.
uint64_t
(
len
(
idsBlob
)),
row_count
:
C
.
int64_t
(
rowCount
),
}
}
/*
/*
CStatus
CStatus
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录