diff --git a/tests/milvus-java-test/README.md b/tests/milvus-java-test/README.md new file mode 100644 index 0000000000000000000000000000000000000000..eba21bb4e472b537382891d02f9b0c5991267502 --- /dev/null +++ b/tests/milvus-java-test/README.md @@ -0,0 +1,29 @@ +# Requirements + +- jdk-1.8 +- testng + +# How to use this Test Project + +1. package and install + +```shell +mvn clean install +``` + +2. start or deploy your milvus server +3. run tests + +```shell +java -cp \"target/MilvusSDkJavaTest-1.0-SNAPSHOT.jar:lib/*\" com.MainClass -h 127.0.0.1 +``` + +4. get test report + +```shell +firefox test-output/index.html +``` + +# Contribution getting started + +Add test cases under testng framework \ No newline at end of file diff --git a/tests/milvus-java-test/pom.xml b/tests/milvus-java-test/pom.xml index db02ff2c009abd07da2051ddbf53835148628300..97956019c0572a96dff3d4d842366e04cbe39080 100644 --- a/tests/milvus-java-test/pom.xml +++ b/tests/milvus-java-test/pom.xml @@ -99,7 +99,7 @@ io.milvus milvus-sdk-java - 0.1.1-SNAPSHOT + 0.1.2-SNAPSHOT @@ -134,4 +134,4 @@ - \ No newline at end of file + diff --git a/tests/milvus-java-test/src/main/java/com/MainClass.java b/tests/milvus-java-test/src/main/java/com/MainClass.java index 0e61e5f3d0fbe5f012a366f514c647a9d5335212..a20916dfb2b1ac5f63f6ea15ed68a2a9ed325e1e 100644 --- a/tests/milvus-java-test/src/main/java/com/MainClass.java +++ b/tests/milvus-java-test/src/main/java/com/MainClass.java @@ -33,7 +33,7 @@ public class MainClass { } @DataProvider(name="ConnectInstance") - public Object[][] connectInstance(){ + public Object[][] connectInstance() throws ConnectFailedException { MilvusClient client = new MilvusGrpcClient(); ConnectParam connectParam = new ConnectParam.Builder() .withHost(host) @@ -45,7 +45,7 @@ public class MainClass { } @DataProvider(name="DisConnectInstance") - public Object[][] disConnectInstance(){ + public Object[][] disConnectInstance() throws ConnectFailedException { // Generate connection instance MilvusClient client = new MilvusGrpcClient(); ConnectParam connectParam = new ConnectParam.Builder() @@ -63,7 +63,7 @@ public class MainClass { } @DataProvider(name="Table") - public Object[][] provideTable(){ + public Object[][] provideTable() throws ConnectFailedException { Object[][] tables = new Object[2][2]; MetricType metricTypes[] = { MetricType.L2, MetricType.IP }; for (Integer i = 0; i < metricTypes.length; ++i) { diff --git a/tests/milvus-java-test/src/main/java/com/TestConnect.java b/tests/milvus-java-test/src/main/java/com/TestConnect.java index 77b6fe6a33dd68cdf43ea38d3aca2e3c199e1ae4..8f6d556f8bee051a50fdb1251da8c97a65cf826f 100644 --- a/tests/milvus-java-test/src/main/java/com/TestConnect.java +++ b/tests/milvus-java-test/src/main/java/com/TestConnect.java @@ -1,15 +1,13 @@ package com; -import io.milvus.client.ConnectParam; -import io.milvus.client.MilvusClient; -import io.milvus.client.MilvusGrpcClient; -import io.milvus.client.Response; +import io.milvus.client.*; +import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; public class TestConnect { @Test(dataProvider = "DefaultConnectArgs", dataProviderClass = MainClass.class) - public void test_connect(String host, String port){ + public void test_connect(String host, String port) throws ConnectFailedException { System.out.println("Host: "+host+", Port: "+port); MilvusClient client = new MilvusGrpcClient(); ConnectParam connectParam = new ConnectParam.Builder() @@ -18,31 +16,42 @@ public class TestConnect { .build(); Response res = client.connect(connectParam); assert(res.ok()); - assert(client.connected()); + assert(client.isConnected()); } @Test(dataProvider = "DefaultConnectArgs", dataProviderClass = MainClass.class) - public void test_connect_repeat(String host, String port){ + public void test_connect_repeat(String host, String port) { MilvusGrpcClient client = new MilvusGrpcClient(); ConnectParam connectParam = new ConnectParam.Builder() .withHost(host) .withPort(port) .build(); - client.connect(connectParam); - Response res = client.connect(connectParam); - assert(!res.ok()); - assert(client.connected()); + Response res = null; + try { + res = client.connect(connectParam); + res = client.connect(connectParam); + } catch (ConnectFailedException e) { + e.printStackTrace(); + } + assert (res.ok()); + assert(client.isConnected()); } @Test(dataProvider="InvalidConnectArgs") - public void test_connect_invalid_connect_args(String ip, String port) throws InterruptedException { + public void test_connect_invalid_connect_args(String ip, String port) { MilvusClient client = new MilvusGrpcClient(); ConnectParam connectParam = new ConnectParam.Builder() .withHost(ip) .withPort(port) .build(); - client.connect(connectParam); - assert(!client.connected()); + Response res = null; + try { + res = client.connect(connectParam); + } catch (ConnectFailedException e) { + e.printStackTrace(); + } + Assert.assertEquals(res, null); + assert(!client.isConnected()); } // TODO: MS-615 @@ -63,18 +72,18 @@ public class TestConnect { @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class) public void test_disconnect(MilvusClient client, String tableName){ - assert(!client.connected()); + assert(!client.isConnected()); } @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class) - public void test_disconnect_repeatably(MilvusClient client, String tableNam){ + public void test_disconnect_repeatably(MilvusClient client, String tableName){ Response res = null; try { res = client.disconnect(); } catch (InterruptedException e) { e.printStackTrace(); } - assert(res.ok()); - assert(!client.connected()); + assert(!res.ok()); + assert(!client.isConnected()); } } diff --git a/tests/milvus-java-test/src/main/java/com/TestIndex.java b/tests/milvus-java-test/src/main/java/com/TestIndex.java index d003771b0b502de014f9f9bf939e6b678eb2871e..5c0493443924937e77b1dc8ba52c3e0dff7301c4 100644 --- a/tests/milvus-java-test/src/main/java/com/TestIndex.java +++ b/tests/milvus-java-test/src/main/java/com/TestIndex.java @@ -140,7 +140,7 @@ public class TestIndex { @Test(dataProvider = "Table", dataProviderClass = MainClass.class) public void test_create_index_IVFSQ8H(MilvusClient client, String tableName) throws InterruptedException { - IndexType indexType = IndexType.IVF_SQ8_H; + IndexType indexType = IndexType.IVF_SQ8H; List> vectors = gen_vectors(nb); InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); client.insert(insertParam); diff --git a/tests/milvus-java-test/src/main/java/com/TestMix.java b/tests/milvus-java-test/src/main/java/com/TestMix.java index 795ab8630ea615108553552dfc2ab9e01a26f686..14ad2e1a3719b957d797949bb8134e99dc0e07e5 100644 --- a/tests/milvus-java-test/src/main/java/com/TestMix.java +++ b/tests/milvus-java-test/src/main/java/com/TestMix.java @@ -71,7 +71,7 @@ public class TestMix { } @Test(dataProvider = "DefaultConnectArgs", dataProviderClass = MainClass.class) - public void test_connect_threads(String host, String port) throws InterruptedException { + public void test_connect_threads(String host, String port) throws ConnectFailedException { int thread_num = 100; ForkJoinPool executor = new ForkJoinPool(); for (int i = 0; i < thread_num; i++) { @@ -82,8 +82,12 @@ public class TestMix { .withHost(host) .withPort(port) .build(); - client.connect(connectParam); - assert(client.connected()); + try { + client.connect(connectParam); + } catch (ConnectFailedException e) { + e.printStackTrace(); + } + assert(client.isConnected()); try { client.disconnect(); } catch (InterruptedException e) { @@ -182,7 +186,7 @@ public class TestMix { } @Test(dataProvider = "DefaultConnectArgs", dataProviderClass = MainClass.class) - public void test_create_insert_delete_threads(String host, String port) throws InterruptedException { + public void test_create_insert_delete_threads(String host, String port) { int thread_num = 100; List> vectors = gen_vectors(nb,false); ForkJoinPool executor = new ForkJoinPool(); @@ -194,7 +198,12 @@ public class TestMix { .withHost(host) .withPort(port) .build(); - client.connect(connectParam); + try { + client.connect(connectParam); + } catch (ConnectFailedException e) { + e.printStackTrace(); + } + assert(client.isConnected()); String tableName = RandomStringUtils.randomAlphabetic(10); TableSchema tableSchema = new TableSchema.Builder(tableName, dimension) .withIndexFileSize(index_file_size) diff --git a/tests/milvus-java-test/src/main/java/com/TestPing.java b/tests/milvus-java-test/src/main/java/com/TestPing.java index 46850f4a17962d9299f70d8d973309792aac0549..1ed462e47de3342d9639cc408fc5b8c134b19fd7 100644 --- a/tests/milvus-java-test/src/main/java/com/TestPing.java +++ b/tests/milvus-java-test/src/main/java/com/TestPing.java @@ -1,14 +1,11 @@ package com; -import io.milvus.client.ConnectParam; -import io.milvus.client.MilvusClient; -import io.milvus.client.MilvusGrpcClient; -import io.milvus.client.Response; +import io.milvus.client.*; import org.testng.annotations.Test; public class TestPing { @Test(dataProvider = "DefaultConnectArgs", dataProviderClass = MainClass.class) - public void test_server_status(String host, String port){ + public void test_server_status(String host, String port) throws ConnectFailedException { System.out.println("Host: "+host+", Port: "+port); MilvusClient client = new MilvusGrpcClient(); ConnectParam connectParam = new ConnectParam.Builder() @@ -16,13 +13,13 @@ public class TestPing { .withPort(port) .build(); client.connect(connectParam); - Response res = client.serverStatus(); + Response res = client.getServerStatus(); assert (res.ok()); } @Test(dataProvider = "DisConnectInstance", dataProviderClass = MainClass.class) public void test_server_status_without_connected(MilvusGrpcClient client, String tableName){ - Response res = client.serverStatus(); + Response res = client.getServerStatus(); assert (!res.ok()); } } \ No newline at end of file diff --git a/tests/milvus_ann_acc/README.md b/tests/milvus_ann_acc/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f5ab9d81682e1712bf78a8b990398d8f035318e2 --- /dev/null +++ b/tests/milvus_ann_acc/README.md @@ -0,0 +1,21 @@ +# Requirements + +- python 3.6+ +- pip install -r requirements.txt + +# How to use this Test Project + +This project is used to test search accuracy based on the given datasets (https://github.com/erikbern/ann-benchmarks#data-sets) + +1. start your milvus server +2. update your test configuration in test.py +3. run command + +```shell +python test.py +``` + +# Contribution getting started + +- Follow PEP-8 for naming and black for formatting. + diff --git a/tests/milvus_ann_acc/requirements.txt b/tests/milvus_ann_acc/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..8c10e71b1f75d8c260768657288c006ab8b60549 --- /dev/null +++ b/tests/milvus_ann_acc/requirements.txt @@ -0,0 +1,4 @@ +numpy==1.16.3 +pymilvus>=0.2.0 +scikit-learn==0.19.1 +h5py==2.7.1 diff --git a/tests/milvus_benchmark/README.md b/tests/milvus_benchmark/README.md index 72abd1264ad2737bd84f5171ebea176868138072..05268057a4b8f6dfc10fa96a24c4b61c63c13d93 100644 --- a/tests/milvus_benchmark/README.md +++ b/tests/milvus_benchmark/README.md @@ -1,57 +1,23 @@ -# Quick start +# Requirements -## 运行 +- python 3.6+ +- pip install -r requirements.txt -### 运行示例： +# How to use this Test Project -`python3 main.py --image=registry.zilliz.com/milvus/engine:branch-0.3.1-release --run-count=2 --run-type=performance` +This project is used to test performance / accuracy / stability of milvus server -### 运行参数： +1. update your test configuration in suites_*.yaml +2. run command ---image: 容器模式，传入镜像名称，如传入，则运行测试时，会先进行pull image，基于image生成milvus server容器 +```shell +### docker mode: +python main.py --image=milvusdb/milvus:latest --run-count=2 --run-type=performance ---local: 与image参数互斥，本地模式，连接使用本地启动的milvus server进行测试 +### local mode: +python main.py --local --run-count=2 --run-type=performance --ip=127.0.0.1 --port=19530 +``` ---run-count: 重复运行次数 +# Contribution getting started ---suites: 测试集配置文件，默认使用suites.yaml - ---run-type: 测试类型，包括性能--performance、准确性测试--accuracy以及稳定性--stability - -### 测试集配置文件： - -`operations: - - insert: - -​ [ -​ {"table.index_type": "ivf_flat", "server.index_building_threshold": 300, "table.size": 2000000, "table.ni": 100000, "table.dim": 512}, -​ ] - - build: [] - - query: - -​ [ -​ {"dataset": "ip_ivfsq8_1000", "top_ks": [10], "nqs": [10, 100], "server.nprobe": 1, "server.use_blas_threshold": 800}, -​ {"dataset": "ip_ivfsq8_1000", "top_ks": [10], "nqs": [10, 100], "server.nprobe": 10, "server.use_blas_threshold": 20}, -​ ]` - -## 测试结果： - -性能： - -`INFO:milvus_benchmark.runner:Start warm query, query params: top-k: 1, nq: 1 - -INFO:milvus_benchmark.client:query run in 19.19s -INFO:milvus_benchmark.runner:Start query, query params: top-k: 64, nq: 10, actually length of vectors: 10 -INFO:milvus_benchmark.runner:Start run query, run 1 of 1 -INFO:milvus_benchmark.client:query run in 0.2s -INFO:milvus_benchmark.runner:Avarage query time: 0.20 -INFO:milvus_benchmark.runner:[[0.2]]` - -**│ 10 │ 0.2 │** - -准确率： - -`INFO:milvus_benchmark.runner:Avarage accuracy: 1.0` \ No newline at end of file +- Follow PEP-8 for naming and black for formatting. \ No newline at end of file diff --git a/tests/milvus_python_test/README.md b/tests/milvus_python_test/README.md index 69b4384d4c682dc9553ca85cdbf566c5f60d33bf..6a87bf1ff8c7946b8cf4b7085e033e06e8c3b546 100644 --- a/tests/milvus_python_test/README.md +++ b/tests/milvus_python_test/README.md @@ -1,14 +1,23 @@ # Requirements -* python 3.6.8 +* python 3.6.8+ +* pip install -r requirements.txt # How to use this Test Project ```shell -pytest . -q -v - ``` +pytest . --level=1 +``` +or test connect function only + +```shell +pytest test_connect.py --level=1 +``` + with allure test report + ```shell pytest --alluredir=test_out . -q -v allure serve test_out ``` # Contribution getting started -* Follow PEP-8 for naming and black for formatting. \ No newline at end of file +* Follow PEP-8 for naming and black for formatting. +