Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
qq_34331598
milvus
提交
4ccf5bac
milvus
项目概览
qq_34331598
/
milvus
与 Fork 源项目一致
Fork自
milvus / milvus
通知
2
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
4ccf5bac
编写于
11月 26, 2019
作者:
J
Jin Hai
提交者:
GitHub
11月 26, 2019
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #538 from XuPeng-SH/track_0.6.0
(shards): fix mysql backend broken issue
上级
f34cdad3
06394c39
变更
10
显示空白变更内容
内联
并排
Showing
10 changed file
with
233 addition
and
5 deletion
+233
-5
.gitignore
.gitignore
+3
-0
shards/.dockerignore
shards/.dockerignore
+1
-0
shards/Makefile
shards/Makefile
+7
-1
shards/all_in_one_with_mysql/all_in_one.yml
shards/all_in_one_with_mysql/all_in_one.yml
+77
-0
shards/all_in_one_with_mysql/mysqld.cnf
shards/all_in_one_with_mysql/mysqld.cnf
+28
-0
shards/all_in_one_with_mysql/probe_test.py
shards/all_in_one_with_mysql/probe_test.py
+25
-0
shards/all_in_one_with_mysql/ro_server.yml
shards/all_in_one_with_mysql/ro_server.yml
+42
-0
shards/all_in_one_with_mysql/wr_server.yml
shards/all_in_one_with_mysql/wr_server.yml
+41
-0
shards/mishards/db_base.py
shards/mishards/db_base.py
+6
-1
shards/mishards/service_handler.py
shards/mishards/service_handler.py
+3
-3
未找到文件。
.gitignore
浏览文件 @
4ccf5bac
...
...
@@ -29,3 +29,6 @@ cmake_build
.coverage
*.pyc
cov_html/
# temp
shards/all_in_one_with_mysql/metadata/
shards/.dockerignore
浏览文件 @
4ccf5bac
...
...
@@ -11,3 +11,4 @@ __pycache__
*.md
*.yml
*.yaml
*/metadata/
shards/Makefile
浏览文件 @
4ccf5bac
...
...
@@ -13,6 +13,12 @@ clean_deploy:
cd
all_in_one
&&
docker-compose
-f
all_in_one.yml down
&&
cd
-
probe_deploy
:
docker run
--rm
--name
probe
--net
=
host milvusdb/mishards /bin/bash
-c
"python all_in_one/probe_test.py"
deploy_m
:
clean_deploy_m
cd
all_in_one_with_mysql
&&
docker-compose
-f
all_in_one.yml up
-d
&&
cd
-
clean_deploy_m
:
cd
all_in_one_with_mysql
&&
docker-compose
-f
all_in_one.yml down
&&
cd
-
probe_deploy_m
:
docker run
--rm
--name
probe
--net
=
host milvusdb/mishards /bin/bash
-c
"python all_in_one_with_mysql/probe_test.py"
cluster
:
cd
kubernetes_demo
;
./start.sh baseup
;
sleep
10
;
./start.sh appup
;
cd
-
clean_cluster
:
...
...
@@ -26,7 +32,7 @@ probe:
docker run
--rm
--name
probe
--net
=
host milvusdb/mishards /bin/bash
-c
"python all_in_one/probe_test.py --port=
${PORT}
--host=
${HOST}
"
clean_coverage
:
rm
-rf
cov_html
clean
:
clean_coverage clean_deploy clean_cluster
clean
:
clean_coverage clean_deploy clean_cluster
clean_deploy_m
style
:
pycodestyle
--config
=
.
coverage
:
...
...
shards/all_in_one_with_mysql/all_in_one.yml
0 → 100644
浏览文件 @
4ccf5bac
version
:
"
2.3"
services
:
milvus-mysql
:
restart
:
always
image
:
mysql:5.7
volumes
:
-
./mysqld.cnf:/etc/mysql/mysql.conf.d/mysqld.cnf
-
./metadata:/var/lib/mysql
environment
:
MYSQL_ROOT_PASSWORD
:
'
milvusroot'
MYSQL_DATABASE
:
'
milvus'
healthcheck
:
test
:
[
"
CMD"
,
"
sleep"
,
"
5"
]
interval
:
1s
timeout
:
10s
retries
:
2
milvus_wr
:
runtime
:
nvidia
restart
:
always
image
:
milvusdb/milvus
volumes
:
-
/tmp/milvus/db:/opt/milvus/db
-
./wr_server.yml:/opt/milvus/conf/server_config.yaml
depends_on
:
milvus-mysql
:
condition
:
service_healthy
milvus_ro
:
runtime
:
nvidia
restart
:
always
image
:
milvusdb/milvus
volumes
:
-
/tmp/milvus/db:/opt/milvus/db
-
./ro_server.yml:/opt/milvus/conf/server_config.yaml
depends_on
:
-
milvus-mysql
-
milvus_wr
jaeger
:
restart
:
always
image
:
jaegertracing/all-in-one:1.14
ports
:
-
"
0.0.0.0:5775:5775/udp"
-
"
0.0.0.0:16686:16686"
-
"
0.0.0.0:9441:9441"
environment
:
COLLECTOR_ZIPKIN_HTTP_PORT
:
9411
mishards
:
restart
:
always
image
:
milvusdb/mishards
ports
:
-
"
0.0.0.0:19531:19531"
-
"
0.0.0.0:19532:19532"
volumes
:
-
/tmp/milvus/db:/tmp/milvus/db
# - /tmp/mishards_env:/source/mishards/.env
command
:
[
"
python"
,
"
mishards/main.py"
]
environment
:
FROM_EXAMPLE
:
'
true'
SQLALCHEMY_DATABASE_URI
:
mysql+pymysql://root:milvusroot@milvus-mysql:3306/milvus?charset=utf8mb4
DEBUG
:
'
true'
SERVER_PORT
:
19531
WOSERVER
:
tcp://milvus_wr:19530
DISCOVERY_PLUGIN_PATH
:
static
DISCOVERY_STATIC_HOSTS
:
milvus_wr,milvus_ro
TRACER_CLASS_NAME
:
jaeger
TRACING_SERVICE_NAME
:
mishards-demo
TRACING_REPORTING_HOST
:
jaeger
TRACING_REPORTING_PORT
:
5775
depends_on
:
-
milvus_wr
-
milvus_ro
-
milvus-mysql
-
jaeger
shards/all_in_one_with_mysql/mysqld.cnf
0 → 100644
浏览文件 @
4ccf5bac
[mysqld]
pid-file = /var/run/mysqld/mysqld.pid
socket = /var/run/mysqld/mysqld.sock
datadir = /var/lib/mysql
log-error = /var/log/mysql/error.log
bind-address = 0.0.0.0
symbolic-links=0
character-set-server = utf8mb4
collation-server = utf8mb4_unicode_ci
init_connect='SET NAMES utf8mb4'
skip-character-set-client-handshake = true
max_connections = 1000
wait_timeout = 31536000
table_open_cache = 128
external-locking = FALSE
binlog_cache_size = 1M
max_heap_table_size = 8M
tmp_table_size = 16M
read_rnd_buffer_size = 8M
sort_buffer_size = 8M
join_buffer_size = 8M
thread_cache_size = 32
query_cache_size = 64M
innodb_buffer_pool_size = 64M
innodb_flush_log_at_trx_commit = 0
innodb_log_buffer_size = 2M
max_allowed_packet=64M
explicit_defaults_for_timestamp=true
\ No newline at end of file
shards/all_in_one_with_mysql/probe_test.py
0 → 100644
浏览文件 @
4ccf5bac
from
milvus
import
Milvus
RED
=
'
\033
[0;31m'
GREEN
=
'
\033
[0;32m'
ENDC
=
''
def
test
(
host
=
'127.0.0.1'
,
port
=
19531
):
client
=
Milvus
()
try
:
status
=
client
.
connect
(
host
=
host
,
port
=
port
)
if
status
.
OK
():
print
(
'{}Pass: Connected{}'
.
format
(
GREEN
,
ENDC
))
return
0
else
:
print
(
'{}Error: {}{}'
.
format
(
RED
,
status
,
ENDC
))
return
1
except
Exception
as
exc
:
print
(
'{}Error: {}{}'
.
format
(
RED
,
exc
,
ENDC
))
return
1
if
__name__
==
'__main__'
:
import
fire
fire
.
Fire
(
test
)
shards/all_in_one_with_mysql/ro_server.yml
0 → 100644
浏览文件 @
4ccf5bac
server_config
:
address
:
0.0.0.0
# milvus server ip address (IPv4)
port
:
19530
# port range: 1025 ~ 65534
deploy_mode
:
cluster_readonly
# deployment type: single, cluster_readonly, cluster_writable
time_zone
:
UTC+8
db_config
:
primary_path
:
/opt/milvus
# path used to store data and meta
secondary_path
:
# path used to store data only, split by semicolon
backend_url
:
mysql://root:milvusroot@milvus-mysql:3306/milvus
# URI format
:
dialect://username:password@host:port/database
# Keep 'dialect://:@:/', and replace other texts with real values
# Replace 'dialect' with 'mysql' or 'sqlite'
insert_buffer_size
:
1
# GB, maximum insert buffer size allowed
# sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory
preload_table
:
# preload data at startup, '*' means load all tables, empty value means no preload
# you can specify preload tables like this: table1,table2,table3
metric_config
:
enable_monitor
:
false
# enable monitoring or not
collector
:
prometheus
# prometheus
prometheus_config
:
port
:
8080
# port prometheus uses to fetch metrics
cache_config
:
cpu_cache_capacity
:
4
# GB, CPU memory used for cache
cpu_cache_threshold
:
0.85
# percentage of data that will be kept when cache cleanup is triggered
gpu_cache_capacity
:
1
# GB, GPU memory used for cache
gpu_cache_threshold
:
0.85
# percentage of data that will be kept when cache cleanup is triggered
cache_insert_data
:
false
# whether to load inserted data into cache
engine_config
:
use_blas_threshold
:
800
# if nq < use_blas_threshold, use SSE, faster with fluctuated response times
# if nq >= use_blas_threshold, use OpenBlas, slower with stable response times
resource_config
:
search_resources
:
# define the GPUs used for search computation, valid value: gpux
-
gpu0
index_build_device
:
gpu0
# GPU used for building index
shards/all_in_one_with_mysql/wr_server.yml
0 → 100644
浏览文件 @
4ccf5bac
server_config
:
address
:
0.0.0.0
# milvus server ip address (IPv4)
port
:
19530
# port range: 1025 ~ 65534
deploy_mode
:
cluster_writable
# deployment type: single, cluster_readonly, cluster_writable
time_zone
:
UTC+8
db_config
:
primary_path
:
/opt/milvus
# path used to store data and meta
secondary_path
:
# path used to store data only, split by semicolon
backend_url
:
mysql://root:milvusroot@milvus-mysql:3306/milvus
# URI format: dialect://username:password@host:port/database
# Keep 'dialect://:@:/', and replace other texts with real values
# Replace 'dialect' with 'mysql' or 'sqlite'
insert_buffer_size
:
2
# GB, maximum insert buffer size allowed
# sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory
preload_table
:
# preload data at startup, '*' means load all tables, empty value means no preload
# you can specify preload tables like this: table1,table2,table3
metric_config
:
enable_monitor
:
false
# enable monitoring or not
collector
:
prometheus
# prometheus
prometheus_config
:
port
:
8080
# port prometheus uses to fetch metrics
cache_config
:
cpu_cache_capacity
:
2
# GB, CPU memory used for cache
cpu_cache_threshold
:
0.85
# percentage of data that will be kept when cache cleanup is triggered
gpu_cache_capacity
:
2
# GB, GPU memory used for cache
gpu_cache_threshold
:
0.85
# percentage of data that will be kept when cache cleanup is triggered
cache_insert_data
:
false
# whether to load inserted data into cache
engine_config
:
use_blas_threshold
:
800
# if nq < use_blas_threshold, use SSE, faster with fluctuated response times
# if nq >= use_blas_threshold, use OpenBlas, slower with stable response times
resource_config
:
search_resources
:
# define the GPUs used for search computation, valid value: gpux
-
gpu0
index_build_device
:
gpu0
# GPU used for building index
shards/mishards/db_base.py
浏览文件 @
4ccf5bac
...
...
@@ -28,7 +28,12 @@ class DB:
if
url
.
get_backend_name
()
==
'sqlite'
:
self
.
engine
=
create_engine
(
url
)
else
:
self
.
engine
=
create_engine
(
uri
,
pool_size
,
pool_recycle
,
pool_timeout
,
pool_pre_ping
,
echo
,
max_overflow
)
self
.
engine
=
create_engine
(
uri
,
pool_size
=
pool_size
,
pool_recycle
=
pool_size
,
pool_timeout
=
pool_timeout
,
pool_pre_ping
=
pool_pre_ping
,
echo
=
echo
,
max_overflow
=
max_overflow
)
self
.
uri
=
uri
self
.
url
=
url
...
...
shards/mishards/service_handler.py
浏览文件 @
4ccf5bac
...
...
@@ -49,7 +49,7 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer):
status
=
status_pb2
.
Status
(
error_code
=
status_pb2
.
SUCCESS
,
reason
=
"Success"
)
if
not
files_n_topk_results
:
return
status
,
[]
return
status
,
[]
,
[]
merge_id_results
=
[]
merge_dis_results
=
[]
...
...
@@ -58,7 +58,7 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer):
for
files_collection
in
files_n_topk_results
:
if
isinstance
(
files_collection
,
tuple
):
status
,
_
=
files_collection
return
status
,
[]
return
status
,
[]
,
[]
row_num
=
files_collection
.
row_num
ids
=
files_collection
.
ids
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录