Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
qq_35024513
milvus
提交
c6a1b300
milvus
项目概览
qq_35024513
/
milvus
与 Fork 源项目一致
Fork自
milvus / milvus
通知
14
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
c6a1b300
编写于
3月 24, 2019
作者:
J
jinhai
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Update test and some interface
上级
3da09021
变更
4
隐藏空白更改
内联
并排
Showing
4 changed file
with
41 addition
and
95 deletion
+41
-95
pyengine/engine/__init__.py
pyengine/engine/__init__.py
+1
-1
pyengine/engine/controller/group_handler.py
pyengine/engine/controller/group_handler.py
+2
-3
pyengine/engine/controller/index_manager.py
pyengine/engine/controller/index_manager.py
+0
-77
pyengine/engine/controller/vector_engine.py
pyengine/engine/controller/vector_engine.py
+38
-14
未找到文件。
pyengine/engine/__init__.py
浏览文件 @
c6a1b300
...
...
@@ -13,4 +13,4 @@ db = SQLAlchemy(app)
from
engine.model.group_table
import
GroupTable
from
engine.model.file_table
import
FileTable
from
engine.controller
import
index_manager
from
engine.controller
import
views
pyengine/engine/controller/group_handler.py
浏览文件 @
c6a1b300
...
...
@@ -10,8 +10,7 @@ class GroupHandler(object):
path
=
path
.
rstrip
(
"
\\
"
)
if
not
os
.
path
.
exists
(
path
):
os
.
makedirs
(
path
)
print
(
"CreateGroupDirectory, Path: "
,
path
)
return
path
@
staticmethod
def
DeleteGroupDirectory
(
group_id
):
...
...
@@ -20,7 +19,7 @@ class GroupHandler(object):
path
=
path
.
rstrip
(
"
\\
"
)
if
os
.
path
.
exists
(
path
):
shutil
.
rmtree
(
path
)
print
(
"DeleteGroupDirectory, Path: "
,
path
)
return
path
@
staticmethod
def
GetGroupDirectory
(
group_id
):
...
...
pyengine/engine/controller/index_manager.py
已删除
100644 → 0
浏览文件 @
3da09021
from
flask
import
Flask
,
jsonify
,
request
from
flask_restful
import
Resource
,
Api
from
engine
import
app
,
db
from
engine.model.group_table
import
GroupTable
from
engine.controller.vector_engine
import
VectorEngine
# app = Flask(__name__)
api
=
Api
(
app
)
from
flask_restful
import
reqparse
from
flask_restful
import
request
class
Vector
(
Resource
):
def
__init__
(
self
):
self
.
__parser
=
reqparse
.
RequestParser
()
self
.
__parser
.
add_argument
(
'vector'
,
type
=
float
,
action
=
'append'
,
location
=
[
'json'
])
def
post
(
self
,
group_id
):
args
=
self
.
__parser
.
parse_args
()
vector
=
args
[
'vector'
]
return
VectorEngine
.
AddVector
(
group_id
,
vector
)
class
VectorSearch
(
Resource
):
def
__init__
(
self
):
self
.
__parser
=
reqparse
.
RequestParser
()
self
.
__parser
.
add_argument
(
'vector'
,
type
=
float
,
action
=
'append'
,
location
=
[
'json'
])
self
.
__parser
.
add_argument
(
'limit'
,
type
=
int
,
action
=
'append'
,
location
=
[
'json'
])
def
post
(
self
,
group_id
):
args
=
self
.
__parser
.
parse_args
()
print
(
'vector: '
,
args
[
'vector'
])
# go to search every thing
return
VectorEngine
.
SearchVector
(
group_id
,
args
[
'vector'
],
args
[
'limit'
])
class
Index
(
Resource
):
def
__init__
(
self
):
self
.
__parser
=
reqparse
.
RequestParser
()
# self.__parser.add_argument('group_id', type=str)
def
post
(
self
,
group_id
):
return
VectorEngine
.
CreateIndex
(
group_id
)
class
Group
(
Resource
):
def
__init__
(
self
):
self
.
__parser
=
reqparse
.
RequestParser
()
self
.
__parser
.
add_argument
(
'group_id'
,
type
=
str
)
self
.
__parser
.
add_argument
(
'dimension'
,
type
=
int
,
action
=
'append'
,
location
=
[
'json'
])
def
post
(
self
,
group_id
):
args
=
self
.
__parser
.
parse_args
()
dimension
=
args
[
'dimension'
]
return
VectorEngine
.
AddGroup
(
group_id
,
dimension
)
def
get
(
self
,
group_id
):
return
VectorEngine
.
GetGroup
(
group_id
)
def
delete
(
self
,
group_id
):
return
VectorEngine
.
DeleteGroup
(
group_id
)
class
GroupList
(
Resource
):
def
get
(
self
):
return
VectorEngine
.
GetGroupList
()
api
.
add_resource
(
Vector
,
'/vector/add/<group_id>'
)
api
.
add_resource
(
Group
,
'/vector/group/<group_id>'
)
api
.
add_resource
(
GroupList
,
'/vector/group'
)
api
.
add_resource
(
Index
,
'/vector/index/<group_id>'
)
api
.
add_resource
(
VectorSearch
,
'/vector/search/<group_id>'
)
# if __name__ == '__main__':
# app.run()
pyengine/engine/controller/vector_engine.py
浏览文件 @
c6a1b300
...
...
@@ -10,13 +10,17 @@ import sys, os
class
VectorEngine
(
object
):
group_dict
=
None
SUCCESS_CODE
=
0
FAULT_CODE
=
1
GROUP_NOT_EXIST
=
2
@
staticmethod
def
AddGroup
(
group_id
,
dimension
):
group
=
GroupTable
.
query
.
filter
(
GroupTable
.
group_name
==
group_id
).
first
()
if
group
:
print
(
'Already create the group: '
,
group_id
)
return
jsonify
({
'code'
:
1
,
'group_name'
:
group_id
,
'file_number'
:
group
.
file_number
})
return
VectorEngine
.
FAULT_CODE
,
group_id
,
group
.
file_number
# return jsonify({'code': 1, 'group_name': group_id, 'file_number': group.file_number})
else
:
print
(
'To create the group: '
,
group_id
)
new_group
=
GroupTable
(
group_id
,
dimension
)
...
...
@@ -25,18 +29,16 @@ class VectorEngine(object):
# add into database
db
.
session
.
add
(
new_group
)
db
.
session
.
commit
()
return
jsonify
({
'code'
:
0
,
'group_name'
:
group_id
,
'file_number'
:
0
})
return
VectorEngine
.
SUCCESS_CODE
,
group_id
,
0
@
staticmethod
def
GetGroup
(
group_id
):
group
=
GroupTable
.
query
.
filter
(
GroupTable
.
group_name
==
group_id
).
first
()
if
group
:
print
(
'Found the group: '
,
group_id
)
return
jsonify
({
'code'
:
0
,
'group_name'
:
group_id
,
'file_number'
:
group
.
file_number
})
return
VectorEngine
.
SUCCESS_CODE
,
group_id
,
group
.
file_number
else
:
print
(
'Not found the group: '
,
group_id
)
return
jsonify
({
'code'
:
1
,
'group_name'
:
group_id
,
'file_number'
:
0
})
# not found
return
VectorEngine
.
FAULT_CODE
,
group_id
,
0
@
staticmethod
...
...
@@ -54,9 +56,9 @@ class VectorEngine(object):
db
.
session
.
delete
(
record
)
db
.
session
.
commit
()
return
jsonify
({
'code'
:
0
,
'group_name'
:
group_id
,
'file_number'
:
group
.
file_number
})
return
VectorEngine
.
SUCCESS_CODE
,
group_id
,
group
.
file_number
else
:
return
jsonify
({
'code'
:
0
,
'group_name'
:
group_id
,
'file_number'
:
0
})
return
VectorEngine
.
SUCCESS_CODE
,
group_id
,
0
@
staticmethod
...
...
@@ -70,12 +72,16 @@ class VectorEngine(object):
group_list
.
append
(
group_item
)
print
(
group_list
)
return
jsonify
(
results
=
group_list
)
return
VectorEngine
.
SUCCESS_CODE
,
group_list
@
staticmethod
def
AddVector
(
group_id
,
vector
):
print
(
group_id
,
vector
)
code
,
_
,
_
=
VectorEngine
.
GetGroup
(
group_id
)
if
code
==
VectorEngine
.
FAULT_CODE
:
return
VectorEngine
.
GROUP_NOT_EXIST
file
=
FileTable
.
query
.
filter
(
FileTable
.
group_name
==
group_id
).
filter
(
FileTable
.
type
==
'raw'
).
first
()
if
file
:
print
(
'insert into exist file'
)
...
...
@@ -112,11 +118,16 @@ class VectorEngine(object):
db
.
session
.
add
(
FileTable
(
group_id
,
raw_filename
,
'raw'
,
1
))
db
.
session
.
commit
()
return
jsonify
({
'code'
:
0
})
return
VectorEngine
.
SUCCESS_CODE
@
staticmethod
def
SearchVector
(
group_id
,
vector
,
limit
):
# Check the group exist
code
,
_
,
_
=
VectorEngine
.
GetGroup
(
group_id
)
if
code
==
VectorEngine
.
FAULT_CODE
:
return
VectorEngine
.
GROUP_NOT_EXIST
# find all files
files
=
FileTable
.
query
.
filter
(
FileTable
.
group_name
==
group_id
).
all
()
...
...
@@ -127,26 +138,31 @@ class VectorEngine(object):
# train
# get topk
print
(
'search in raw file: '
,
file
.
filename
)
pass
else
:
# get topk
print
(
'search in index file: '
,
file
.
filename
)
data
=
IndexFileHandler
.
Read
(
file
.
filename
,
file
.
type
)
pass
# according to difference files get topk of each
# reduce the topk from them
# construct response and send back
return
jsonify
({
'code'
:
0
})
vector_id
=
0
return
VectorEngine
.
SUCCESS_CODE
,
vector_id
@
staticmethod
def
CreateIndex
(
group_id
):
# Check the group exist
code
,
_
,
_
=
VectorEngine
.
GetGroup
(
group_id
)
if
code
==
VectorEngine
.
FAULT_CODE
:
return
VectorEngine
.
GROUP_NOT_EXIST
# create index
file
=
FileTable
.
query
.
filter
(
FileTable
.
group_name
==
group_id
).
filter
(
FileTable
.
type
==
'raw'
).
first
()
path
=
GroupHandler
.
GetGroupDirectory
(
group_id
)
+
'/'
+
file
.
filename
print
(
'Going to create index for: '
,
path
)
return
jsonify
({
'code'
:
0
})
return
VectorEngine
.
SUCCESS_CODE
@
staticmethod
...
...
@@ -156,6 +172,8 @@ class VectorEngine(object):
if
VectorEngine
.
group_dict
is
None
:
# print("VectorEngine.group_dict is None")
VectorEngine
.
group_dict
=
dict
()
if
not
(
group_id
in
VectorEngine
.
group_dict
):
VectorEngine
.
group_dict
[
group_id
]
=
[]
VectorEngine
.
group_dict
[
group_id
].
append
(
vector
)
...
...
@@ -174,3 +192,9 @@ class VectorEngine(object):
def
GetVectorListFromRawFile
(
group_id
,
filename
):
return
VectorEngine
.
group_dict
[
group_id
]
@
staticmethod
def
ClearRawFile
(
group_id
):
print
(
"VectorEngine.group_dict: "
,
VectorEngine
.
group_dict
)
del
VectorEngine
.
group_dict
[
group_id
]
return
VectorEngine
.
SUCCESS_CODE
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录