Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
qq_35024513
milvus
提交
dbfb4f20
milvus
项目概览
qq_35024513
/
milvus
与 Fork 源项目一致
Fork自
milvus / milvus
通知
14
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
dbfb4f20
编写于
3月 25, 2019
作者:
X
xj.lin
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
add list => array
上级
5616ec74
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
13 addition
and
18 deletion
+13
-18
pyengine/engine/controller/scheduler.py
pyengine/engine/controller/scheduler.py
+7
-14
pyengine/engine/controller/tests/test_vector_engine.py
pyengine/engine/controller/tests/test_vector_engine.py
+4
-1
pyengine/engine/controller/vector_engine.py
pyengine/engine/controller/vector_engine.py
+2
-3
未找到文件。
pyengine/engine/controller/scheduler.py
浏览文件 @
dbfb4f20
from
engine.retrieval
import
search_index
from
engine.retrieval
import
search_index
from
engine.ingestion
import
build_index
from
engine.ingestion
import
build_index
from
engine.ingestion
import
serialize
from
engine.ingestion
import
serialize
import
numpy
as
np
class
Singleton
(
type
):
class
Singleton
(
type
):
_instances
=
{}
_instances
=
{}
...
@@ -15,36 +15,29 @@ class Scheduler(metaclass=Singleton):
...
@@ -15,36 +15,29 @@ class Scheduler(metaclass=Singleton):
def
Search
(
self
,
index_file_key
,
vectors
,
k
):
def
Search
(
self
,
index_file_key
,
vectors
,
k
):
# assert index_file_key
# assert index_file_key
# assert vectors
# assert vectors
# assert k
assert
k
!=
0
query_vectors
=
serialize
.
to_array
(
vectors
)
return
self
.
__scheduler
(
index_file_key
,
vectors
,
k
)
return
self
.
__scheduler
(
index_file_key
,
query_
vectors
,
k
)
def
__scheduler
(
self
,
index_data_key
,
vectors
,
k
):
def
__scheduler
(
self
,
index_data_key
,
vectors
,
k
):
result_list
=
[]
result_list
=
[]
d
=
None
raw_vectors
=
None
print
(
"__scheduler: vectors: "
,
vectors
)
query_vectors
=
np
.
asarray
(
vectors
).
astype
(
'float32'
)
if
'raw'
in
index_data_key
:
if
'raw'
in
index_data_key
:
raw_vectors
=
index_data_key
[
'raw'
]
raw_vectors
=
index_data_key
[
'raw'
]
raw_vectors
=
np
.
asarray
(
raw_vectors
).
astype
(
'float32'
)
d
=
index_data_key
[
'dimension'
]
d
=
index_data_key
[
'dimension'
]
if
'raw'
in
index_data_key
:
index_builder
=
build_index
.
FactoryIndex
()
index_builder
=
build_index
.
FactoryIndex
()
print
(
"d: "
,
d
,
" raw_vectors: "
,
raw_vectors
)
index
=
index_builder
().
build
(
d
,
raw_vectors
)
index
=
index_builder
().
build
(
d
,
raw_vectors
)
searcher
=
search_index
.
FaissSearch
(
index
)
searcher
=
search_index
.
FaissSearch
(
index
)
result_list
.
append
(
searcher
.
search_by_vectors
(
query_
vectors
,
k
))
result_list
.
append
(
searcher
.
search_by_vectors
(
vectors
,
k
))
index_data_list
=
index_data_key
[
'index'
]
index_data_list
=
index_data_key
[
'index'
]
for
key
in
index_data_list
:
for
key
in
index_data_list
:
index
=
GetIndexData
(
key
)
index
=
GetIndexData
(
key
)
searcher
=
search_index
.
FaissSearch
(
index
)
searcher
=
search_index
.
FaissSearch
(
index
)
result_list
.
append
(
searcher
.
search_by_vectors
(
query_
vectors
,
k
))
result_list
.
append
(
searcher
.
search_by_vectors
(
vectors
,
k
))
if
len
(
result_list
)
==
1
:
if
len
(
result_list
)
==
1
:
return
result_list
[
0
].
vectors
return
result_list
[
0
].
vectors
...
...
pyengine/engine/controller/tests/test_vector_engine.py
浏览文件 @
dbfb4f20
...
@@ -3,6 +3,7 @@ from engine.settings import DATABASE_DIRECTORY
...
@@ -3,6 +3,7 @@ from engine.settings import DATABASE_DIRECTORY
from
flask
import
jsonify
from
flask
import
jsonify
import
pytest
import
pytest
import
os
import
os
import
numpy
as
np
import
logging
import
logging
logging
.
basicConfig
(
level
=
logging
.
INFO
,
format
=
'%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logging
.
basicConfig
(
level
=
logging
.
INFO
,
format
=
'%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
...
@@ -104,10 +105,12 @@ class TestVectorEngine:
...
@@ -104,10 +105,12 @@ class TestVectorEngine:
expected_list
=
[
self
.
__vector
]
expected_list
=
[
self
.
__vector
]
vector_list
=
VectorEngine
.
GetVectorListFromRawFile
(
'test_group'
,
filename
)
vector_list
=
VectorEngine
.
GetVectorListFromRawFile
(
'test_group'
,
filename
)
print
(
'expected_list: '
,
expected_list
)
print
(
'expected_list: '
,
expected_list
)
print
(
'vector_list: '
,
vector_list
)
print
(
'vector_list: '
,
vector_list
)
expected_list
=
np
.
asarray
(
expected_list
).
astype
(
'float32'
)
assert
vector_list
==
expected_list
assert
np
.
all
(
vector_list
==
expected_list
)
code
=
VectorEngine
.
ClearRawFile
(
'test_group'
)
code
=
VectorEngine
.
ClearRawFile
(
'test_group'
)
assert
code
==
VectorEngine
.
SUCCESS_CODE
assert
code
==
VectorEngine
.
SUCCESS_CODE
...
...
pyengine/engine/controller/vector_engine.py
浏览文件 @
dbfb4f20
...
@@ -144,7 +144,7 @@ class VectorEngine(object):
...
@@ -144,7 +144,7 @@ class VectorEngine(object):
index_keys
=
[
i
.
filename
for
i
in
files
if
i
.
type
==
'index'
]
index_keys
=
[
i
.
filename
for
i
in
files
if
i
.
type
==
'index'
]
index_map
=
{}
index_map
=
{}
index_map
[
'index'
]
=
index_keys
index_map
[
'index'
]
=
index_keys
index_map
[
'raw'
]
=
VectorEngine
.
GetVectorListFromRawFile
(
group_id
,
"fakename"
)
index_map
[
'raw'
]
=
VectorEngine
.
GetVectorListFromRawFile
(
group_id
,
"fakename"
)
#TODO: pass by key, get from storage
index_map
[
'dimension'
]
=
group
.
dimension
index_map
[
'dimension'
]
=
group
.
dimension
scheduler_instance
=
Scheduler
()
scheduler_instance
=
Scheduler
()
...
@@ -188,8 +188,7 @@ class VectorEngine(object):
...
@@ -188,8 +188,7 @@ class VectorEngine(object):
@
staticmethod
@
staticmethod
def
GetVectorListFromRawFile
(
group_id
,
filename
=
"todo"
):
def
GetVectorListFromRawFile
(
group_id
,
filename
=
"todo"
):
return
VectorEngine
.
group_dict
[
group_id
]
return
serialize
.
to_array
(
VectorEngine
.
group_dict
[
group_id
])
# return serialize.to_array(VectorEngine.group_dict[group_id])
@
staticmethod
@
staticmethod
def
ClearRawFile
(
group_id
):
def
ClearRawFile
(
group_id
):
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录