Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
af1900b4
M
milvus
项目概览
milvus
/
milvus
11 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
af1900b4
编写于
3月 09, 2021
作者:
F
FluorineDog
提交者:
yefu.chen
3月 09, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Remove ConcurrentBitsetPtr in segcore
Signed-off-by:
N
FluorineDog
<
guilin.gou@zilliz.com
>
上级
455cc59d
变更
11
隐藏空白更改
内联
并排
Showing
11 changed file
with
111 addition
and
30 deletion
+111
-30
internal/core/src/query/CMakeLists.txt
internal/core/src/query/CMakeLists.txt
+1
-1
internal/core/src/query/SearchOnGrowing.cpp
internal/core/src/query/SearchOnGrowing.cpp
+0
-14
internal/core/src/query/SearchOnGrowing.h
internal/core/src/query/SearchOnGrowing.h
+2
-2
internal/core/src/query/SearchOnSealed.cpp
internal/core/src/query/SearchOnSealed.cpp
+13
-11
internal/core/src/query/SearchOnSealed.h
internal/core/src/query/SearchOnSealed.h
+1
-1
internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp
internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp
+1
-1
internal/core/thirdparty/CMakeLists.txt
internal/core/thirdparty/CMakeLists.txt
+1
-0
internal/core/thirdparty/boost_ext/CMakeLists.txt
internal/core/thirdparty/boost_ext/CMakeLists.txt
+2
-0
internal/core/thirdparty/boost_ext/LICENSE
internal/core/thirdparty/boost_ext/LICENSE
+21
-0
internal/core/thirdparty/boost_ext/dynamic_bitset_ext.cpp
internal/core/thirdparty/boost_ext/dynamic_bitset_ext.cpp
+63
-0
internal/core/thirdparty/boost_ext/dynamic_bitset_ext.hpp
internal/core/thirdparty/boost_ext/dynamic_bitset_ext.hpp
+6
-0
未找到文件。
internal/core/src/query/CMakeLists.txt
浏览文件 @
af1900b4
...
...
@@ -18,4 +18,4 @@ set(MILVUS_QUERY_SRCS
SubQueryResult.cpp
)
add_library
(
milvus_query
${
MILVUS_QUERY_SRCS
}
)
target_link_libraries
(
milvus_query milvus_proto milvus_utils knowhere
)
target_link_libraries
(
milvus_query milvus_proto milvus_utils knowhere
boost_bitset_ext
)
internal/core/src/query/SearchOnGrowing.cpp
浏览文件 @
af1900b4
...
...
@@ -20,20 +20,6 @@
#include "query/SearchOnIndex.h"
namespace
milvus
::
query
{
static
faiss
::
ConcurrentBitsetPtr
create_bitmap_view
(
std
::
optional
<
const
BitmapSimple
*>
bitmaps_opt
,
int64_t
chunk_id
)
{
if
(
!
bitmaps_opt
.
has_value
())
{
return
nullptr
;
}
auto
&
bitmaps
=
*
bitmaps_opt
.
value
();
auto
src_vec
=
~
bitmaps
.
at
(
chunk_id
);
auto
dst
=
std
::
make_shared
<
faiss
::
ConcurrentBitset
>
(
src_vec
.
size
());
auto
iter
=
reinterpret_cast
<
BitmapChunk
::
block_type
*>
(
dst
->
mutable_data
());
boost
::
to_block_range
(
src_vec
,
iter
);
return
dst
;
}
Status
FloatSearch
(
const
segcore
::
SegmentGrowingImpl
&
segment
,
const
query
::
QueryInfo
&
info
,
...
...
internal/core/src/query/SearchOnGrowing.h
浏览文件 @
af1900b4
...
...
@@ -17,8 +17,8 @@
#include "query/SubQueryResult.h"
namespace
milvus
::
query
{
using
Bit
map
Chunk
=
boost
::
dynamic_bitset
<>
;
using
Bit
mapSimple
=
std
::
deque
<
Bitmap
Chunk
>
;
using
Bit
set
Chunk
=
boost
::
dynamic_bitset
<>
;
using
Bit
setSimple
=
std
::
deque
<
Bitset
Chunk
>
;
void
SearchOnGrowing
(
const
segcore
::
SegmentGrowingImpl
&
segment
,
...
...
internal/core/src/query/SearchOnSealed.cpp
浏览文件 @
af1900b4
...
...
@@ -17,32 +17,34 @@
#include <knowhere/index/vector_index/VecIndex.h>
#include "knowhere/index/vector_index/helpers/IndexParameter.h"
#include "knowhere/index/vector_index/adapter/VectorAdapter.h"
#include <boost_ext/dynamic_bitset_ext.hpp>
namespace
milvus
::
query
{
// negate bitset, and merge them into one
aligned_vector
<
uint8_t
>
AssembleNegBit
map
(
const
BitmapSimple
&
bitmap
_simple
)
{
AssembleNegBit
set
(
const
BitsetSimple
&
bitset
_simple
)
{
int64_t
N
=
0
;
for
(
auto
&
bit
map
:
bitmap
_simple
)
{
N
+=
bit
map
.
size
();
for
(
auto
&
bit
set
:
bitset
_simple
)
{
N
+=
bit
set
.
size
();
}
aligned_vector
<
uint8_t
>
result
(
upper_align
(
upper_div
(
N
,
8
),
sizeof
(
BitmapChunk
::
block_type
)
));
aligned_vector
<
uint8_t
>
result
(
upper_align
(
upper_div
(
N
,
8
),
64
));
auto
acc_byte_count
=
0
;
for
(
auto
&
bitmap_raw
:
bitmap_simple
)
{
auto
bitmap
=
~
bitmap_raw
;
auto
size
=
bitmap
.
size
();
for
(
auto
&
bitset
:
bitset_simple
)
{
auto
size
=
bitset
.
size
();
Assert
(
size
%
8
==
0
);
auto
byte_count
=
size
/
8
;
auto
iter
=
reinterpret_cast
<
BitmapChunk
::
block_type
*>
(
result
.
data
()
+
acc_byte_count
);
boost
::
to_block_range
(
bitmap
,
iter
);
auto
src_ptr
=
boost_ext
::
get_data
(
bitset
);
memcpy
(
result
.
data
()
+
acc_byte_count
,
src_ptr
,
byte_count
);
acc_byte_count
+=
byte_count
;
}
// revert the bitset
for
(
int64_t
i
=
0
;
i
<
result
.
size
();
++
i
)
{
result
[
i
]
=
~
result
[
i
];
}
return
result
;
}
...
...
internal/core/src/query/SearchOnSealed.h
浏览文件 @
af1900b4
...
...
@@ -18,7 +18,7 @@
namespace
milvus
::
query
{
aligned_vector
<
uint8_t
>
AssembleNegBit
map
(
const
Bitmap
Simple
&
bitmap_simple
);
AssembleNegBit
set
(
const
Bitset
Simple
&
bitmap_simple
);
void
SearchOnSealed
(
const
Schema
&
schema
,
...
...
internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp
浏览文件 @
af1900b4
...
...
@@ -80,7 +80,7 @@ ExecPlanNodeVisitor::VectorVisitorImpl(VectorPlanNode& node) {
if
(
node
.
predicate_
.
has_value
())
{
ExecExprVisitor
::
RetType
expr_ret
=
ExecExprVisitor
(
*
segment
,
row_count
).
call_child
(
*
node
.
predicate_
.
value
());
bitset_holder
=
AssembleNegBit
map
(
expr_ret
);
bitset_holder
=
AssembleNegBit
set
(
expr_ret
);
view
=
BitsetView
(
bitset_holder
.
data
(),
bitset_holder
.
size
()
*
8
);
}
...
...
internal/core/thirdparty/CMakeLists.txt
浏览文件 @
af1900b4
...
...
@@ -55,3 +55,4 @@ endif()
add_subdirectory
(
protobuf
)
add_subdirectory
(
fiu
)
add_subdirectory
(
boost_ext
)
internal/core/thirdparty/boost_ext/CMakeLists.txt
0 → 100644
浏览文件 @
af1900b4
find_package
(
Boost REQUIRED
)
add_library
(
boost_bitset_ext dynamic_bitset_ext.cpp
)
internal/core/thirdparty/boost_ext/LICENSE
0 → 100644
浏览文件 @
af1900b4
MIT License
Copyright (c) 2021 FluorineDog
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
internal/core/thirdparty/boost_ext/dynamic_bitset_ext.cpp
0 → 100644
浏览文件 @
af1900b4
#include <iostream>
#include "dynamic_bitset_ext.hpp"
namespace
{
struct
PtrWrapper
{
explicit
PtrWrapper
(
char
*&
ptr
)
:
ptr_
(
ptr
)
{}
char
*&
ptr_
;
};
struct
ConstPtrWrapper
{
explicit
ConstPtrWrapper
(
const
char
*&
ptr
)
:
ptr_
(
ptr
)
{}
const
char
*&
ptr_
;
};
using
Block
=
unsigned
long
;
using
Allocator
=
std
::
allocator
<
Block
>
;
}
// namespace
namespace
boost
{
// a language lawyer's way to steal original pointer from boost::dynamic_bitset
// salute to http://www.gotw.ca/gotw/076.htm
template
<
>
void
from_block_range
<
PtrWrapper
,
Block
,
Allocator
>
(
PtrWrapper
result
,
PtrWrapper
resultB
,
dynamic_bitset
<>&
bitset
)
{
(
void
)
resultB
;
result
.
ptr_
=
reinterpret_cast
<
char
*>
(
bitset
.
m_bits
.
data
());
}
template
<
>
void
to_block_range
<
Block
,
Allocator
,
ConstPtrWrapper
>
(
const
dynamic_bitset
<>&
bitset
,
ConstPtrWrapper
result
)
{
result
.
ptr_
=
reinterpret_cast
<
const
char
*>
(
bitset
.
m_bits
.
data
());
}
}
// namespace boost
namespace
boost_ext
{
char
*
get_data
(
boost
::
dynamic_bitset
<>&
bitset
)
{
char
*
ptr
=
nullptr
;
PtrWrapper
wrapper
{
ptr
};
boost
::
from_block_range
(
wrapper
,
wrapper
,
bitset
);
assert
(
ptr
);
return
ptr
;
}
const
char
*
get_data
(
const
boost
::
dynamic_bitset
<>&
bitset
)
{
const
char
*
ptr
=
nullptr
;
ConstPtrWrapper
wrapper
{
ptr
};
boost
::
to_block_range
(
bitset
,
wrapper
);
assert
(
ptr
);
return
ptr
;
}
}
// namespace boost_ext
internal/core/thirdparty/boost_ext/dynamic_bitset_ext.hpp
0 → 100644
浏览文件 @
af1900b4
#include <boost/dynamic_bitset.hpp>
namespace
boost_ext
{
const
char
*
get_data
(
const
boost
::
dynamic_bitset
<>&
bitset
);
char
*
get_data
(
boost
::
dynamic_bitset
<>&
bitset
);
}
// namespace boost_ext
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录