diff --git a/.gitignore b/.gitignore index 8fda9f2980540e3d46d2e8b55d36449f68aab0ac..0a120ebe959e7d39e565a9049e676e7f7583a0cb 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,6 @@ cmake_build .coverage *.pyc cov_html/ + +# temp +shards/all_in_one_with_mysql/metadata/ diff --git a/shards/.dockerignore b/shards/.dockerignore index e4506100574e38a83892af1c161155cd6a42b66e..cb7486f242c1536eca0b34a218698cd650e3d975 100644 --- a/shards/.dockerignore +++ b/shards/.dockerignore @@ -11,3 +11,4 @@ __pycache__ *.md *.yml *.yaml +*/metadata/ diff --git a/shards/Makefile b/shards/Makefile index c8aa6127f8f020969c7e018b681edf611822518e..5127882d6ff5afc8ca8be987c3107874a2d02781 100644 --- a/shards/Makefile +++ b/shards/Makefile @@ -13,6 +13,12 @@ clean_deploy: cd all_in_one && docker-compose -f all_in_one.yml down && cd - probe_deploy: docker run --rm --name probe --net=host milvusdb/mishards /bin/bash -c "python all_in_one/probe_test.py" +deploy_m: clean_deploy_m + cd all_in_one_with_mysql && docker-compose -f all_in_one.yml up -d && cd - +clean_deploy_m: + cd all_in_one_with_mysql && docker-compose -f all_in_one.yml down && cd - +probe_deploy_m: + docker run --rm --name probe --net=host milvusdb/mishards /bin/bash -c "python all_in_one_with_mysql/probe_test.py" cluster: cd kubernetes_demo;./start.sh baseup;sleep 10;./start.sh appup;cd - clean_cluster: @@ -26,7 +32,7 @@ probe: docker run --rm --name probe --net=host milvusdb/mishards /bin/bash -c "python all_in_one/probe_test.py --port=${PORT} --host=${HOST}" clean_coverage: rm -rf cov_html -clean: clean_coverage clean_deploy clean_cluster +clean: clean_coverage clean_deploy clean_cluster clean_deploy_m style: pycodestyle --config=. coverage: diff --git a/shards/all_in_one_with_mysql/all_in_one.yml b/shards/all_in_one_with_mysql/all_in_one.yml new file mode 100644 index 0000000000000000000000000000000000000000..6619635f2874243593fed823ddef47dfcc7cec63 --- /dev/null +++ b/shards/all_in_one_with_mysql/all_in_one.yml @@ -0,0 +1,77 @@ +version: "2.3" +services: + milvus-mysql: + restart: always + image: mysql:5.7 + volumes: + - ./mysqld.cnf:/etc/mysql/mysql.conf.d/mysqld.cnf + - ./metadata:/var/lib/mysql + environment: + MYSQL_ROOT_PASSWORD: 'milvusroot' + MYSQL_DATABASE: 'milvus' + healthcheck: + test: ["CMD", "sleep", "5"] + interval: 1s + timeout: 10s + retries: 2 + + milvus_wr: + runtime: nvidia + restart: always + image: milvusdb/milvus + volumes: + - /tmp/milvus/db:/opt/milvus/db + - ./wr_server.yml:/opt/milvus/conf/server_config.yaml + depends_on: + milvus-mysql: + condition: service_healthy + + milvus_ro: + runtime: nvidia + restart: always + image: milvusdb/milvus + volumes: + - /tmp/milvus/db:/opt/milvus/db + - ./ro_server.yml:/opt/milvus/conf/server_config.yaml + depends_on: + - milvus-mysql + - milvus_wr + + jaeger: + restart: always + image: jaegertracing/all-in-one:1.14 + ports: + - "0.0.0.0:5775:5775/udp" + - "0.0.0.0:16686:16686" + - "0.0.0.0:9441:9441" + environment: + COLLECTOR_ZIPKIN_HTTP_PORT: 9411 + + mishards: + restart: always + image: milvusdb/mishards + ports: + - "0.0.0.0:19531:19531" + - "0.0.0.0:19532:19532" + volumes: + - /tmp/milvus/db:/tmp/milvus/db + # - /tmp/mishards_env:/source/mishards/.env + command: ["python", "mishards/main.py"] + environment: + FROM_EXAMPLE: 'true' + SQLALCHEMY_DATABASE_URI: mysql+pymysql://root:milvusroot@milvus-mysql:3306/milvus?charset=utf8mb4 + DEBUG: 'true' + SERVER_PORT: 19531 + WOSERVER: tcp://milvus_wr:19530 + DISCOVERY_PLUGIN_PATH: static + DISCOVERY_STATIC_HOSTS: milvus_wr,milvus_ro + TRACER_CLASS_NAME: jaeger + TRACING_SERVICE_NAME: mishards-demo + TRACING_REPORTING_HOST: jaeger + TRACING_REPORTING_PORT: 5775 + + depends_on: + - milvus_wr + - milvus_ro + - milvus-mysql + - jaeger diff --git a/shards/all_in_one_with_mysql/mysqld.cnf b/shards/all_in_one_with_mysql/mysqld.cnf new file mode 100644 index 0000000000000000000000000000000000000000..5f636767577ca776be4cbf0beb1a2da357cdbe94 --- /dev/null +++ b/shards/all_in_one_with_mysql/mysqld.cnf @@ -0,0 +1,28 @@ +[mysqld] +pid-file = /var/run/mysqld/mysqld.pid +socket = /var/run/mysqld/mysqld.sock +datadir = /var/lib/mysql +log-error = /var/log/mysql/error.log +bind-address = 0.0.0.0 +symbolic-links=0 +character-set-server = utf8mb4 +collation-server = utf8mb4_unicode_ci +init_connect='SET NAMES utf8mb4' +skip-character-set-client-handshake = true +max_connections = 1000 +wait_timeout = 31536000 +table_open_cache = 128 +external-locking = FALSE +binlog_cache_size = 1M +max_heap_table_size = 8M +tmp_table_size = 16M +read_rnd_buffer_size = 8M +sort_buffer_size = 8M +join_buffer_size = 8M +thread_cache_size = 32 +query_cache_size = 64M +innodb_buffer_pool_size = 64M +innodb_flush_log_at_trx_commit = 0 +innodb_log_buffer_size = 2M +max_allowed_packet=64M +explicit_defaults_for_timestamp=true \ No newline at end of file diff --git a/shards/all_in_one_with_mysql/probe_test.py b/shards/all_in_one_with_mysql/probe_test.py new file mode 100644 index 0000000000000000000000000000000000000000..62504659100f88158a8ed368f2f878f10c9fd0b7 --- /dev/null +++ b/shards/all_in_one_with_mysql/probe_test.py @@ -0,0 +1,25 @@ +from milvus import Milvus + +RED = '\033[0;31m' +GREEN = '\033[0;32m' +ENDC = '' + + +def test(host='127.0.0.1', port=19531): + client = Milvus() + try: + status = client.connect(host=host, port=port) + if status.OK(): + print('{}Pass: Connected{}'.format(GREEN, ENDC)) + return 0 + else: + print('{}Error: {}{}'.format(RED, status, ENDC)) + return 1 + except Exception as exc: + print('{}Error: {}{}'.format(RED, exc, ENDC)) + return 1 + + +if __name__ == '__main__': + import fire + fire.Fire(test) diff --git a/shards/all_in_one_with_mysql/ro_server.yml b/shards/all_in_one_with_mysql/ro_server.yml new file mode 100644 index 0000000000000000000000000000000000000000..768fafb1a021a6a8ded6c37eed616fc60f590782 --- /dev/null +++ b/shards/all_in_one_with_mysql/ro_server.yml @@ -0,0 +1,42 @@ +server_config: + address: 0.0.0.0 # milvus server ip address (IPv4) + port: 19530 # port range: 1025 ~ 65534 + deploy_mode: cluster_readonly # deployment type: single, cluster_readonly, cluster_writable + time_zone: UTC+8 + +db_config: + primary_path: /opt/milvus # path used to store data and meta + secondary_path: # path used to store data only, split by semicolon + + backend_url: mysql://root:milvusroot@milvus-mysql:3306/milvus + # URI format: dialect://username:password@host:port/database + # Keep 'dialect://:@:/', and replace other texts with real values + # Replace 'dialect' with 'mysql' or 'sqlite' + + insert_buffer_size: 1 # GB, maximum insert buffer size allowed + # sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory + + preload_table: # preload data at startup, '*' means load all tables, empty value means no preload + # you can specify preload tables like this: table1,table2,table3 + +metric_config: + enable_monitor: false # enable monitoring or not + collector: prometheus # prometheus + prometheus_config: + port: 8080 # port prometheus uses to fetch metrics + +cache_config: + cpu_cache_capacity: 4 # GB, CPU memory used for cache + cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered + gpu_cache_capacity: 1 # GB, GPU memory used for cache + gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered + cache_insert_data: false # whether to load inserted data into cache + +engine_config: + use_blas_threshold: 800 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times + # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times + +resource_config: + search_resources: # define the GPUs used for search computation, valid value: gpux + - gpu0 + index_build_device: gpu0 # GPU used for building index diff --git a/shards/all_in_one_with_mysql/wr_server.yml b/shards/all_in_one_with_mysql/wr_server.yml new file mode 100644 index 0000000000000000000000000000000000000000..b2332532e54f16b8a551ba466e308df3787e76bc --- /dev/null +++ b/shards/all_in_one_with_mysql/wr_server.yml @@ -0,0 +1,41 @@ +server_config: + address: 0.0.0.0 # milvus server ip address (IPv4) + port: 19530 # port range: 1025 ~ 65534 + deploy_mode: cluster_writable # deployment type: single, cluster_readonly, cluster_writable + time_zone: UTC+8 + +db_config: + primary_path: /opt/milvus # path used to store data and meta + secondary_path: # path used to store data only, split by semicolon + + backend_url: mysql://root:milvusroot@milvus-mysql:3306/milvus # URI format: dialect://username:password@host:port/database + # Keep 'dialect://:@:/', and replace other texts with real values + # Replace 'dialect' with 'mysql' or 'sqlite' + + insert_buffer_size: 2 # GB, maximum insert buffer size allowed + # sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory + + preload_table: # preload data at startup, '*' means load all tables, empty value means no preload + # you can specify preload tables like this: table1,table2,table3 + +metric_config: + enable_monitor: false # enable monitoring or not + collector: prometheus # prometheus + prometheus_config: + port: 8080 # port prometheus uses to fetch metrics + +cache_config: + cpu_cache_capacity: 2 # GB, CPU memory used for cache + cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered + gpu_cache_capacity: 2 # GB, GPU memory used for cache + gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered + cache_insert_data: false # whether to load inserted data into cache + +engine_config: + use_blas_threshold: 800 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times + # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times + +resource_config: + search_resources: # define the GPUs used for search computation, valid value: gpux + - gpu0 + index_build_device: gpu0 # GPU used for building index diff --git a/shards/mishards/db_base.py b/shards/mishards/db_base.py index e55c330352488ea27cf13d2ffb961f958cb70247..6754f5c80a6d59660cc78c68f9509a3ad1f64a2b 100644 --- a/shards/mishards/db_base.py +++ b/shards/mishards/db_base.py @@ -28,7 +28,12 @@ class DB: if url.get_backend_name() == 'sqlite': self.engine = create_engine(url) else: - self.engine = create_engine(uri, pool_size, pool_recycle, pool_timeout, pool_pre_ping, echo, max_overflow) + self.engine = create_engine(uri, pool_size=pool_size, + pool_recycle=pool_size, + pool_timeout=pool_timeout, + pool_pre_ping=pool_pre_ping, + echo=echo, + max_overflow=max_overflow) self.uri = uri self.url = url diff --git a/shards/mishards/service_handler.py b/shards/mishards/service_handler.py index fc0ee0fa2b660a50fd92d1aee21eb62e5eb5be83..f192aff72c7839f389fbdcf6a0d0fb95509c5e92 100644 --- a/shards/mishards/service_handler.py +++ b/shards/mishards/service_handler.py @@ -49,7 +49,7 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): status = status_pb2.Status(error_code=status_pb2.SUCCESS, reason="Success") if not files_n_topk_results: - return status, [] + return status, [], [] merge_id_results = [] merge_dis_results = [] @@ -58,8 +58,8 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): for files_collection in files_n_topk_results: if isinstance(files_collection, tuple): status, _ = files_collection - return status, [] - + return status, [], [] + row_num = files_collection.row_num ids = files_collection.ids diss = files_collection.distances # distance collections