Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
14ef405a
M
milvus
项目概览
milvus
/
milvus
接近 1 年 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
14ef405a
编写于
9月 15, 2021
作者:
C
Cai Yudong
提交者:
GitHub
9月 15, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Rename chunk_size to chunk_rows for better code readability (#7953)
Signed-off-by:
N
yudong.cai
<
yudong.cai@zilliz.com
>
上级
ab84f95e
变更
13
隐藏空白更改
内联
并排
Showing
13 changed file
with
31 addition
and
31 deletion
+31
-31
configs/advanced/query_node.yaml
configs/advanced/query_node.yaml
+1
-1
internal/core/bench/bench_search.cpp
internal/core/bench/bench_search.cpp
+2
-2
internal/core/src/segcore/FieldIndexing.h
internal/core/src/segcore/FieldIndexing.h
+1
-1
internal/core/src/segcore/SegcoreConfig.cpp
internal/core/src/segcore/SegcoreConfig.cpp
+2
-2
internal/core/src/segcore/SegcoreConfig.h
internal/core/src/segcore/SegcoreConfig.h
+5
-5
internal/core/src/segcore/SegmentGrowingImpl.cpp
internal/core/src/segcore/SegmentGrowingImpl.cpp
+6
-6
internal/core/src/segcore/SegmentGrowingImpl.h
internal/core/src/segcore/SegmentGrowingImpl.h
+2
-2
internal/core/src/segcore/segcore_init_c.cpp
internal/core/src/segcore/segcore_init_c.cpp
+3
-3
internal/core/src/segcore/segcore_init_c.h
internal/core/src/segcore/segcore_init_c.h
+1
-1
internal/core/unittest/test_init.cpp
internal/core/unittest/test_init.cpp
+1
-1
internal/core/unittest/test_utils/test_segcore.yaml
internal/core/unittest/test_utils/test_segcore.yaml
+1
-1
internal/querynode/param_table.go
internal/querynode/param_table.go
+4
-4
internal/querynode/query_node.go
internal/querynode/query_node.go
+2
-2
未找到文件。
configs/advanced/query_node.yaml
浏览文件 @
14ef405a
...
...
@@ -27,5 +27,5 @@ queryNode:
recvBufSize
:
64
segcore
:
chunk
Size
:
32768
# 32M
chunk
Rows
:
32768
simdType
:
auto
# auto, avx512, avx2, sse
internal/core/bench/bench_search.cpp
浏览文件 @
14ef405a
...
...
@@ -68,9 +68,9 @@ Search_SmallIndex(benchmark::State& state) {
}();
auto
is_small_index
=
state
.
range
(
0
);
auto
chunk_
size
=
state
.
range
(
1
)
*
1024
;
auto
chunk_
rows
=
state
.
range
(
1
)
*
1024
;
auto
segconf
=
SegcoreConfig
::
default_config
();
segconf
.
set_
size_per_chunk
(
chunk_size
);
segconf
.
set_
chunk_rows
(
chunk_rows
);
auto
segment
=
CreateGrowingSegment
(
schema
,
segconf
);
if
(
!
is_small_index
)
{
segment
->
disable_small_index
();
...
...
internal/core/src/segcore/FieldIndexing.h
浏览文件 @
14ef405a
...
...
@@ -45,7 +45,7 @@ class FieldIndexing {
int64_t
get_size_per_chunk
()
const
{
return
segcore_config_
.
get_
size_per_chunk
();
return
segcore_config_
.
get_
chunk_rows
();
}
virtual
knowhere
::
Index
*
...
...
internal/core/src/segcore/SegcoreConfig.cpp
浏览文件 @
14ef405a
...
...
@@ -48,8 +48,8 @@ SegcoreConfig::parse_from(const std::string& config_path) {
YAML
::
Node
top_config
=
YAML
::
LoadFile
(
config_path
);
Assert
(
top_config
.
IsMap
());
auto
seg_config
=
subnode
(
top_config
,
"segcore"
);
auto
chunk_
size
=
subnode
(
seg_config
,
"chunk_size
"
).
as
<
int64_t
>
();
this
->
size_per_chunk_
=
chunk_size
;
auto
chunk_
rows
=
subnode
(
seg_config
,
"chunk_rows
"
).
as
<
int64_t
>
();
this
->
chunk_rows_
=
chunk_rows
;
#if 0
auto index_list = subnode(seg_config, "small_index");
...
...
internal/core/src/segcore/SegcoreConfig.h
浏览文件 @
14ef405a
...
...
@@ -55,13 +55,13 @@ class SegcoreConfig {
}
int64_t
get_
size_per_chunk
()
const
{
return
size_per_chunk
_
;
get_
chunk_rows
()
const
{
return
chunk_rows
_
;
}
void
set_
size_per_chunk
(
int64_t
size_per_chunk
)
{
size_per_chunk_
=
size_per_chunk
;
set_
chunk_rows
(
int64_t
chunk_rows
)
{
chunk_rows_
=
chunk_rows
;
}
void
...
...
@@ -70,7 +70,7 @@ class SegcoreConfig {
}
private:
int64_t
size_per_chunk
_
=
32
*
1024
;
int64_t
chunk_rows
_
=
32
*
1024
;
std
::
map
<
MetricType
,
SmallIndexConf
>
table_
;
};
...
...
internal/core/src/segcore/SegmentGrowingImpl.cpp
浏览文件 @
14ef405a
...
...
@@ -203,8 +203,8 @@ SegmentGrowingImpl::do_insert(int64_t reserved_begin,
record_
.
ack_responder_
.
AddSegment
(
reserved_begin
,
reserved_begin
+
size
);
if
(
enable_small_index_
)
{
int64_t
chunk_
size
=
segcore_config_
.
get_size_per_chunk
();
indexing_record_
.
UpdateResourceAck
(
record_
.
ack_responder_
.
GetAck
()
/
chunk_
size
,
record_
);
int64_t
chunk_
rows
=
segcore_config_
.
get_chunk_rows
();
indexing_record_
.
UpdateResourceAck
(
record_
.
ack_responder_
.
GetAck
()
/
chunk_
rows
,
record_
);
}
}
...
...
@@ -243,10 +243,10 @@ SegmentGrowingImpl::Delete(int64_t reserved_begin,
int64_t
SegmentGrowingImpl
::
GetMemoryUsageInBytes
()
const
{
int64_t
total_bytes
=
0
;
auto
size_per_chunk
=
segcore_config_
.
get_size_per_chunk
();
int64_t
ins_n
=
upper_align
(
record_
.
reserved
,
size_per_chunk
);
auto
chunk_rows
=
segcore_config_
.
get_chunk_rows
();
int64_t
ins_n
=
upper_align
(
record_
.
reserved
,
chunk_rows
);
total_bytes
+=
ins_n
*
(
schema_
->
get_total_sizeof
()
+
16
+
1
);
int64_t
del_n
=
upper_align
(
deleted_record_
.
reserved
,
size_per_chunk
);
int64_t
del_n
=
upper_align
(
deleted_record_
.
reserved
,
chunk_rows
);
total_bytes
+=
del_n
*
(
16
*
2
);
return
total_bytes
;
}
...
...
@@ -260,7 +260,7 @@ SegmentGrowingImpl::chunk_data_impl(FieldOffset field_offset, int64_t chunk_id)
int64_t
SegmentGrowingImpl
::
num_chunk
()
const
{
auto
size
=
get_insert_record
().
ack_responder_
.
GetAck
();
return
upper_div
(
size
,
segcore_config_
.
get_
size_per_chunk
());
return
upper_div
(
size
,
segcore_config_
.
get_
chunk_rows
());
}
void
...
...
internal/core/src/segcore/SegmentGrowingImpl.h
浏览文件 @
14ef405a
...
...
@@ -109,7 +109,7 @@ class SegmentGrowingImpl : public SegmentGrowing {
int64_t
size_per_chunk
()
const
final
{
return
segcore_config_
.
get_
size_per_chunk
();
return
segcore_config_
.
get_
chunk_rows
();
}
public:
...
...
@@ -159,7 +159,7 @@ class SegmentGrowingImpl : public SegmentGrowing {
explicit
SegmentGrowingImpl
(
SchemaPtr
schema
,
const
SegcoreConfig
&
segcore_config
)
:
segcore_config_
(
segcore_config
),
schema_
(
std
::
move
(
schema
)),
record_
(
*
schema_
,
segcore_config
.
get_
size_per_chunk
()),
record_
(
*
schema_
,
segcore_config
.
get_
chunk_rows
()),
indexing_record_
(
*
schema_
,
segcore_config_
)
{
}
...
...
internal/core/src/segcore/segcore_init_c.cpp
浏览文件 @
14ef405a
...
...
@@ -37,10 +37,10 @@ SegcoreInit() {
}
extern
"C"
void
SegcoreSetChunk
Size
(
const
int64_t
value
)
{
SegcoreSetChunk
Rows
(
const
int64_t
value
)
{
milvus
::
segcore
::
SegcoreConfig
&
config
=
milvus
::
segcore
::
SegcoreConfig
::
default_config
();
config
.
set_
size_per_chunk
(
value
);
std
::
cout
<<
"set config chunk_size: "
<<
config
.
get_
size_per_chunk
()
<<
std
::
endl
;
config
.
set_
chunk_rows
(
value
);
std
::
cout
<<
"set config chunk_size: "
<<
config
.
get_
chunk_rows
()
<<
std
::
endl
;
}
extern
"C"
void
...
...
internal/core/src/segcore/segcore_init_c.h
浏览文件 @
14ef405a
...
...
@@ -19,7 +19,7 @@ void
SegcoreInit
();
void
SegcoreSetChunk
Size
(
const
int64_t
);
SegcoreSetChunk
Rows
(
const
int64_t
);
void
SegcoreSetSimdType
(
const
char
*
);
...
...
internal/core/unittest/test_init.cpp
浏览文件 @
14ef405a
...
...
@@ -20,6 +20,6 @@ TEST(Init, Naive) {
using
namespace
milvus
;
using
namespace
milvus
::
segcore
;
SegcoreInit
();
SegcoreSetChunk
Size
(
32768
);
SegcoreSetChunk
Rows
(
32768
);
SegcoreSetSimdType
(
"auto"
);
}
internal/core/unittest/test_utils/test_segcore.yaml
浏览文件 @
14ef405a
segcore
:
chunk_
size
:
32768
chunk_
rows
:
32768
small_index
:
-
metric_type
:
[
"
L2"
,
"
IP"
]
index_type
:
"
IVF"
...
...
internal/querynode/param_table.go
浏览文件 @
14ef405a
...
...
@@ -69,7 +69,7 @@ type ParamTable struct {
SliceIndex
int
// segcore
Chunk
Size
int64
Chunk
Rows
int64
SimdType
string
Log
log
.
Config
...
...
@@ -115,7 +115,7 @@ func (p *ParamTable) Init() {
p
.
initStatsPublishInterval
()
p
.
initStatsChannelName
()
p
.
initSegcoreChunk
Size
()
p
.
initSegcoreChunk
Rows
()
p
.
initSegcoreSimdType
()
p
.
initLogCfg
()
...
...
@@ -261,8 +261,8 @@ func (p *ParamTable) initStatsChannelName() {
p
.
StatsChannelName
=
channels
}
func
(
p
*
ParamTable
)
initSegcoreChunk
Size
()
{
p
.
Chunk
Size
=
p
.
ParseInt64
(
"queryNode.segcore.chunkSize
"
)
func
(
p
*
ParamTable
)
initSegcoreChunk
Rows
()
{
p
.
Chunk
Rows
=
p
.
ParseInt64
(
"queryNode.segcore.chunkRows
"
)
}
func
(
p
*
ParamTable
)
initSegcoreSimdType
()
{
...
...
internal/querynode/query_node.go
浏览文件 @
14ef405a
...
...
@@ -103,8 +103,8 @@ func (node *QueryNode) InitSegcore() {
C
.
SegcoreInit
()
// override segcore chunk size
cChunk
Size
:=
C
.
int64_t
(
Params
.
ChunkSize
)
C
.
SegcoreSetChunk
Size
(
cChunkSize
)
cChunk
Rows
:=
C
.
int64_t
(
Params
.
ChunkRows
)
C
.
SegcoreSetChunk
Rows
(
cChunkRows
)
// override segcore SIMD type
cSimdType
:=
C
.
CString
(
Params
.
SimdType
)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录