From 8f88529fc1f41eb5c9985b40c525f664c9fe280a Mon Sep 17 00:00:00 2001
From: del-zhenwu <56623710+del-zhenwu@users.noreply.github.com>
Date: Fri, 17 Jan 2020 09:59:49 +0800
Subject: [PATCH] Add nightly test notification && improve test case stability 
 (#1056)

* Update framework

* remove files

* Remove files

* Remove ann-acc cases && Update java-sdk cases

* change cn to en

* [skip ci] remove doc test

* [skip ci] change cn to en

* Case stability

* Add mail notification when test failed

* Add main notification

* Add main notification

* gen milvus instance from utils

* Distable case with multiprocess

* Add mail notification when nightly test failed

* add milvus handler param

* add http handler

Co-authored-by: quicksilver <zhifeng.zhang@zilliz.com>
---
 ci/jenkins/Jenkinsfile                        |  18 +
 ci/jenkins/scripts/mail.py                    |  36 ++
 tests/milvus_python_test/conftest.py          |  47 +-
 tests/milvus_python_test/pytest.ini           |   6 +-
 tests/milvus_python_test/test_add_vectors.py  | 129 +-----
 tests/milvus_python_test/test_connect.py      |  41 +-
 .../milvus_python_test/test_delete_vectors.py | 425 ------------------
 tests/milvus_python_test/test_index.py        |  10 +-
 tests/milvus_python_test/test_mix.py          |   8 +-
 tests/milvus_python_test/test_partition.py    |   2 +-
 .../milvus_python_test/test_search_vectors.py |  10 +-
 tests/milvus_python_test/test_table.py        |  10 +-
 tests/milvus_python_test/test_table_count.py  |   9 +-
 tests/milvus_python_test/utils.py             |  19 +-
 14 files changed, 148 insertions(+), 622 deletions(-)
 create mode 100644 ci/jenkins/scripts/mail.py
 delete mode 100644 tests/milvus_python_test/test_delete_vectors.py

diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile
index ad2087f1..54750370 100644
--- a/ci/jenkins/Jenkinsfile
+++ b/ci/jenkins/Jenkinsfile
@@ -189,6 +189,24 @@ pipeline {
             }
         }
     }
+    post {
+        always {
+            script {
+                boolean isNightlyTest = isTimeTriggeredBuild()
+                if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS') && isNightlyTest) {
+                    // Send an email only if the build status has changed from green/unstable to red
+                    emailext subject: '$DEFAULT_SUBJECT',
+                    body: '$DEFAULT_CONTENT',
+                    recipientProviders: [
+                        [$class: 'DevelopersRecipientProvider'],
+                        [$class: 'RequesterRecipientProvider']
+                    ],
+                    replyTo: '$DEFAULT_REPLYTO',
+                    to: '$DEFAULT_RECIPIENTS'
+                }
+            }
+        }
+    }
 }
 
 boolean isTimeTriggeredBuild() {
diff --git a/ci/jenkins/scripts/mail.py b/ci/jenkins/scripts/mail.py
new file mode 100644
index 00000000..960a1898
--- /dev/null
+++ b/ci/jenkins/scripts/mail.py
@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+import sys
+import logging
+from email.mime.text import MIMEText
+from email.header import Header
+import smtplib
+
+SMS_DEFAULT_TO_LIST = [
+    "dev.milvus@zilliz.com",
+]
+
+def send_email(subject, content, token, receivers=None):
+    sender = 'test@zilliz.com'
+    message = MIMEText(content, 'html', 'utf-8')
+    message['From'] = Header("Daily Test")
+    message['To'] = Header("dev.milvus")
+    message['Subject'] = Header(subject, 'utf-8')
+    try:
+        smtp_obj = smtplib.SMTP('smtp.exmail.qq.com')
+        if receivers is None:
+            receivers = SMS_DEFAULT_TO_LIST
+        smtp_obj.login(sender, token)
+        result = smtp_obj.sendmail(sender, receivers, message.as_string())
+    except smtplib.SMTPException as e:
+        logging.error(str(e))
+    finally:
+        smtp_obj.quit()
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 4:
+        sys.exit()
+    subject = sys.argv[1]
+    content = sys.argv[2]
+    token = sys.argv[3]
+    send_email(subject, content, token)
diff --git a/tests/milvus_python_test/conftest.py b/tests/milvus_python_test/conftest.py
index ef932983..bc8ebe15 100644
--- a/tests/milvus_python_test/conftest.py
+++ b/tests/milvus_python_test/conftest.py
@@ -5,6 +5,7 @@ import logging
 import pytest
 from utils import gen_unique_str
 from milvus import Milvus, IndexType, MetricType
+from utils import *
 
 index_file_size = 10
 
@@ -12,12 +13,14 @@ index_file_size = 10
 def pytest_addoption(parser):
     parser.addoption("--ip", action="store", default="localhost")
     parser.addoption("--port", action="store", default=19530)
-    parser.addoption("--internal", action="store", default=False)
+    parser.addoption("--http-port", action="store", default=19121)
+    parser.addoption("--handler", action="store", default="GRPC")
 
 
 def check_server_connection(request):
     ip = request.config.getoption("--ip")
     port = request.config.getoption("--port")
+
     connected = True
     if ip and (ip not in ['localhost', '127.0.0.1']):
         try:
@@ -28,20 +31,16 @@ def check_server_connection(request):
     return connected
 
 
-def get_args(request):
-    args = {
-        "ip": request.config.getoption("--ip"),
-        "port": request.config.getoption("--port")
-    }
-    return args
-
-
 @pytest.fixture(scope="module")
 def connect(request):
     ip = request.config.getoption("--ip")
     port = request.config.getoption("--port")
-    milvus = Milvus()
+    http_port = request.config.getoption("--http-port")
+    handler = request.config.getoption("--handler")
+    milvus = get_milvus(handler=handler)
     try:
+        if handler == "HTTP":
+            port = http_port
         status = milvus.connect(host=ip, port=port)
         logging.getLogger().info(status)
         if not status.OK():
@@ -66,16 +65,9 @@ def connect(request):
 def dis_connect(request):
     ip = request.config.getoption("--ip")
     port = request.config.getoption("--port")
-    milvus = Milvus()
-    milvus.connect(host=ip, port=port)
-    milvus.disconnect()
-    def fin():
-        try:
-            milvus.disconnect()
-        except:
-            pass
-
-    request.addfinalizer(fin)
+    http_port = request.config.getoption("--http-port")
+    handler = request.config.getoption("--handler")
+    milvus = get_milvus(handler=handler)
     return milvus
 
 
@@ -83,13 +75,20 @@ def dis_connect(request):
 def args(request):
     ip = request.config.getoption("--ip")
     port = request.config.getoption("--port")
-    internal = request.config.getoption("--internal")
-    args = {"ip": ip, "port": port}
-    if internal:
-        args = {"ip": ip, "port": port, "internal": internal}
+    http_port = request.config.getoption("--http-port")
+    handler = request.config.getoption("--handler")
+    if handler == "HTTP":
+        port = http_port
+    args = {"ip": ip, "port": port, "handler": handler}
     return args
 
 
+@pytest.fixture(scope="module")
+def milvus(request):
+    handler = request.config.getoption("--handler")
+    return get_milvus(handler=handler)
+
+
 @pytest.fixture(scope="function")
 def table(request, connect):
     ori_table_name = getattr(request.module, "table_id", "test")
diff --git a/tests/milvus_python_test/pytest.ini b/tests/milvus_python_test/pytest.ini
index 3ae6a790..a0188768 100644
--- a/tests/milvus_python_test/pytest.ini
+++ b/tests/milvus_python_test/pytest.ini
@@ -6,4 +6,8 @@ log_level = 20
 
 timeout = 600
 
-level = 1
\ No newline at end of file
+markers = 
+    level: test level
+    serial
+
+#level = 1
diff --git a/tests/milvus_python_test/test_add_vectors.py b/tests/milvus_python_test/test_add_vectors.py
index 015bfb78..e43a6b2c 100644
--- a/tests/milvus_python_test/test_add_vectors.py
+++ b/tests/milvus_python_test/test_add_vectors.py
@@ -5,7 +5,7 @@ import threading
 import logging
 from multiprocessing import Pool, Process
 import pytest
-from milvus import Milvus, IndexType, MetricType
+from milvus import IndexType, MetricType
 from utils import *
 
 
@@ -576,38 +576,8 @@ class TestAddBase:
         assert status.OK()
         assert len(result) == 1
 
-    # @pytest.mark.repeat(5)
-    @pytest.mark.timeout(ADD_TIMEOUT)
-    def _test_add_vector_multi_threading(self, connect, table):
-        '''
-        target: test add vectors, with multi threading
-        method: 10 thread add vectors concurrently
-        expected: status ok and result length is equal to the length off added vectors
-        '''
-        thread_num = 4
-        loops = 100
-        threads = []
-        total_ids = []
-        vector = gen_single_vector(dim)
-        def add():
-            i = 0
-            while i < loops:
-                status, ids = connect.add_vectors(table, vector)
-                total_ids.append(ids[0])
-                i = i + 1
-        for i in range(thread_num):
-            x = threading.Thread(target=add, args=())
-            threads.append(x)
-            x.start()
-            time.sleep(0.2)
-        for th in threads:
-            th.join()
-        assert len(total_ids) == thread_num * loops
-        # make sure ids not the same
-        assert len(set(total_ids)) == thread_num * loops
-
     # TODO: enable
-    # @pytest.mark.repeat(5)
+    # @pytest.mark.repeat(10)
     @pytest.mark.timeout(ADD_TIMEOUT)
     def _test_add_vector_with_multiprocessing(self, args):
         '''
@@ -615,36 +585,35 @@ class TestAddBase:
         method: 10 processed add vectors concurrently
         expected: status ok and result length is equal to the length off added vectors
         '''
-        table = gen_unique_str("test_add_vector_with_multiprocessing")
+        table = gen_unique_str()
         uri = "tcp://%s:%s" % (args["ip"], args["port"])
         param = {'table_name': table,
                  'dimension': dim,
-                 'index_file_size': index_file_size}
-        # create table
-        milvus = Milvus()
+                 'index_file_size': index_file_size,
+                 'metric_type': MetricType.L2}
+        milvus = get_milvus()
         milvus.connect(uri=uri)
         milvus.create_table(param)
         vector = gen_single_vector(dim)
-
         process_num = 4
-        loop_num = 10
+        loop_num = 5
         processes = []
-        # with dependent connection
-        def add(milvus):
+        def add():
+            milvus = get_milvus()
+            milvus.connect(uri=uri)
             i = 0
             while i < loop_num:
                 status, ids = milvus.add_vectors(table, vector)
                 i = i + 1
+            milvus.disconnect()
         for i in range(process_num):
-            milvus = Milvus()
-            milvus.connect(uri=uri)
-            p = Process(target=add, args=(milvus,))
+            p = Process(target=add, args=())
             processes.append(p)
             p.start()
             time.sleep(0.2)
         for p in processes:
             p.join()
-        time.sleep(3)
+        time.sleep(2)
         status, count = milvus.get_table_row_count(table)
         assert count == process_num * loop_num
 
@@ -1149,78 +1118,6 @@ class TestAddIP:
         assert status.OK()
         assert len(result) == 1
 
-    # @pytest.mark.repeat(5)
-    @pytest.mark.timeout(ADD_TIMEOUT)
-    def _test_add_vector_multi_threading(self, connect, ip_table):
-        '''
-        target: test add vectors, with multi threading
-        method: 10 thread add vectors concurrently
-        expected: status ok and result length is equal to the length off added vectors
-        '''
-        thread_num = 4
-        loops = 100
-        threads = []
-        total_ids = []
-        vector = gen_single_vector(dim)
-        def add():
-            i = 0
-            while i < loops:
-                status, ids = connect.add_vectors(ip_table, vector)
-                total_ids.append(ids[0])
-                i = i + 1
-        for i in range(thread_num):
-            x = threading.Thread(target=add, args=())
-            threads.append(x)
-            x.start()
-            time.sleep(0.2)
-        for th in threads:
-            th.join()
-        assert len(total_ids) == thread_num * loops
-        # make sure ids not the same
-        assert len(set(total_ids)) == thread_num * loops
-
-    # TODO: enable
-    # @pytest.mark.repeat(5)
-    @pytest.mark.timeout(ADD_TIMEOUT)
-    def _test_add_vector_with_multiprocessing(self, args):
-        '''
-        target: test add vectors, with multi processes
-        method: 10 processed add vectors concurrently
-        expected: status ok and result length is equal to the length off added vectors
-        '''
-        table = gen_unique_str("test_add_vector_with_multiprocessing")
-        uri = "tcp://%s:%s" % (args["ip"], args["port"])
-        param = {'table_name': table,
-                 'dimension': dim,
-                 'index_file_size': index_file_size}
-        # create table
-        milvus = Milvus()
-        milvus.connect(uri=uri)
-        milvus.create_table(param)
-        vector = gen_single_vector(dim)
-
-        process_num = 4
-        loop_num = 10
-        processes = []
-        # with dependent connection
-        def add(milvus):
-            i = 0
-            while i < loop_num:
-                status, ids = milvus.add_vectors(table, vector)
-                i = i + 1
-        for i in range(process_num):
-            milvus = Milvus()
-            milvus.connect(uri=uri)
-            p = Process(target=add, args=(milvus,))
-            processes.append(p)
-            p.start()
-            time.sleep(0.2)
-        for p in processes:
-            p.join()
-        time.sleep(3)
-        status, count = milvus.get_table_row_count(table)
-        assert count == process_num * loop_num
-
     def test_add_vector_multi_tables(self, connect):
         '''
         target: test add vectors is correct or not with multiple tables of IP
diff --git a/tests/milvus_python_test/test_connect.py b/tests/milvus_python_test/test_connect.py
index 132efcbb..f3a99c7d 100644
--- a/tests/milvus_python_test/test_connect.py
+++ b/tests/milvus_python_test/test_connect.py
@@ -1,5 +1,4 @@
 import pytest
-from milvus import Milvus
 import pdb
 import threading
 from multiprocessing import Process
@@ -37,7 +36,7 @@ class TestConnect:
         expected: raise an error after disconnected
         '''
         if not connect.connected():
-            milvus = Milvus()
+            milvus = get_milvus()
             uri_value = "tcp://%s:%s" % (args["ip"], args["port"])
             milvus.connect(uri=uri_value)
             res = milvus.disconnect()
@@ -54,7 +53,7 @@ class TestConnect:
         method: set correct ip and port
         expected: connected is True        
         '''
-        milvus = Milvus()
+        milvus = get_milvus()
         milvus.connect(host=args["ip"], port=args["port"])
         assert milvus.connected()
 
@@ -64,7 +63,7 @@ class TestConnect:
         method: set correct ip and port
         expected: connected is False        
         '''
-        milvus = Milvus()
+        milvus = get_milvus()
         milvus.connect(host=args["ip"], port=args["port"])
         milvus.disconnect()
         assert not milvus.connected()
@@ -76,7 +75,7 @@ class TestConnect:
         method: set host localhost
         expected: connected is True
         '''
-        milvus = Milvus()
+        milvus = get_milvus()
         milvus.connect(host='localhost', port=args["port"])
         assert milvus.connected()
 
@@ -87,7 +86,7 @@ class TestConnect:
         method: set host null
         expected: not use default ip, connected is False
         '''
-        milvus = Milvus()
+        milvus = get_milvus()
         ip = ""
         with pytest.raises(Exception) as e:
             milvus.connect(host=ip, port=args["port"], timeout=1)
@@ -99,7 +98,7 @@ class TestConnect:
         method: uri format and value are both correct
         expected: connected is True        
         '''
-        milvus = Milvus()
+        milvus = get_milvus()
         uri_value = "tcp://%s:%s" % (args["ip"], args["port"])
         milvus.connect(uri=uri_value)
         assert milvus.connected()
@@ -110,7 +109,7 @@ class TestConnect:
         method: uri set null
         expected: connected is True        
         '''
-        milvus = Milvus()
+        milvus = get_milvus()
         uri_value = ""
         
         if self.local_ip(args):
@@ -129,7 +128,7 @@ class TestConnect:
         method: set uri port null
         expected: connected is True        
         '''
-        milvus = Milvus()
+        milvus = get_milvus()
         uri_value = "tcp://%s:" % args["ip"]
         with pytest.raises(Exception) as e:
             milvus.connect(uri=uri_value, timeout=1)
@@ -142,7 +141,7 @@ class TestConnect:
         method: set uri ip null
         expected: connected is True        
         '''
-        milvus = Milvus()
+        milvus = get_milvus()
         uri_value = "tcp://:%s" % args["port"]
         
         with pytest.raises(Exception) as e:
@@ -167,7 +166,7 @@ class TestConnect:
             assert milvus.connected()
 
         for i in range(process_num):
-            milvus = Milvus()
+            milvus = get_milvus()
             p = Process(target=connect, args=(milvus, ))
             processes.append(p)
             p.start()
@@ -180,7 +179,7 @@ class TestConnect:
         method: connect again
         expected: status.code is 0, and status.message shows have connected already
         '''
-        milvus = Milvus()
+        milvus = get_milvus()
         uri_value = "tcp://%s:%s" % (args["ip"], args["port"])
         milvus.connect(uri=uri_value)
         
@@ -193,7 +192,7 @@ class TestConnect:
         method: disconnect, and then connect, assert connect status
         expected: status.code is 0
         '''
-        milvus = Milvus()
+        milvus = get_milvus()
         uri_value = "tcp://%s:%s" % (args["ip"], args["port"])
         milvus.connect(uri=uri_value)
         
@@ -208,7 +207,7 @@ class TestConnect:
         expected: status.code is 0
         '''
         times = 10
-        milvus = Milvus()
+        milvus = get_milvus()
         uri_value = "tcp://%s:%s" % (args["ip"], args["port"])
         milvus.connect(uri=uri_value)
         for i in range(times):
@@ -234,7 +233,7 @@ class TestConnect:
             assert milvus.connected()
 
         for i in range(process_num):
-            milvus = Milvus()
+            milvus = get_milvus()
             p = Process(target=connect, args=(milvus, ))
             processes.append(p)
             p.start()
@@ -247,7 +246,7 @@ class TestConnect:
         method: port set "", check if wrong uri connection is ok
         expected: connect raise an exception and connected is false
         '''
-        milvus = Milvus()
+        milvus = get_milvus()
         uri_value = "tcp://%s:39540" % args["ip"]
         with pytest.raises(Exception) as e:
             milvus.connect(host=args["ip"], port="", uri=uri_value)
@@ -258,7 +257,7 @@ class TestConnect:
         method: host set "", check if correct uri connection is ok
         expected: connected is False
         '''
-        milvus = Milvus()
+        milvus = get_milvus()
         uri_value = "tcp://%s:%s" % (args["ip"], args["port"])
         with pytest.raises(Exception) as e:
             milvus.connect(host="", port=args["port"], uri=uri_value, timeout=1)
@@ -271,7 +270,7 @@ class TestConnect:
         method: check if wrong uri connection is ok
         expected: connect raise an exception and connected is false
         '''
-        milvus = Milvus()
+        milvus = get_milvus()
         uri_value = "tcp://%s:%s" % (args["ip"], args["port"])
         with pytest.raises(Exception) as e:
             res = milvus.connect(host=args["ip"], port=39540, uri=uri_value, timeout=1)
@@ -327,7 +326,7 @@ class TestConnectIPInvalid(object):
     @pytest.mark.level(2)
     @pytest.mark.timeout(CONNECT_TIMEOUT)
     def test_connect_with_invalid_ip(self, args, get_invalid_ip):
-        milvus = Milvus()
+        milvus = get_milvus()
         ip = get_invalid_ip
         with pytest.raises(Exception) as e:
             milvus.connect(host=ip, port=args["port"], timeout=1)
@@ -354,7 +353,7 @@ class TestConnectPortInvalid(object):
         method: set port in gen_invalid_ports
         expected: connected is False        
         '''
-        milvus = Milvus()
+        milvus = get_milvus()
         port = get_invalid_port
         with pytest.raises(Exception) as e:
             milvus.connect(host=args["ip"], port=port, timeout=1)
@@ -380,7 +379,7 @@ class TestConnectURIInvalid(object):
         method: set port in gen_invalid_uris
         expected: connected is False        
         '''
-        milvus = Milvus()
+        milvus = get_milvus()
         uri_value = get_invalid_uri
         with pytest.raises(Exception) as e:
             milvus.connect(uri=uri_value, timeout=1)
diff --git a/tests/milvus_python_test/test_delete_vectors.py b/tests/milvus_python_test/test_delete_vectors.py
deleted file mode 100644
index 57ab7677..00000000
--- a/tests/milvus_python_test/test_delete_vectors.py
+++ /dev/null
@@ -1,425 +0,0 @@
-# import time
-# import random
-# import pdb
-# import logging
-# import threading
-# from builtins import Exception
-# from multiprocessing import Pool, Process
-# import pytest
-
-# from milvus import Milvus, IndexType
-# from utils import *
-
-
-# dim = 128
-# index_file_size = 10
-# table_id = "test_delete"
-# DELETE_TIMEOUT = 60
-# vectors = gen_vectors(100, dim)
-
-# class TestDeleteVectorsBase:
-#     """
-#     generate invalid query range params
-#     """
-#     @pytest.fixture(
-#         scope="function",
-#         params=[
-#             (get_current_day(), get_current_day()),
-#             (get_last_day(1), get_last_day(1)),
-#             (get_next_day(1), get_next_day(1))
-#         ]
-#     )
-#     def get_invalid_range(self, request):
-#         yield request.param
-
-#     @pytest.mark.timeout(DELETE_TIMEOUT)
-#     def test_delete_vectors_invalid_range(self, connect, table, get_invalid_range):
-#         '''
-#         target: test delete vectors, no index created
-#         method: call `delete_vectors_by_range`, with invalid date params
-#         expected: return code 0
-#         '''
-#         start_date = get_invalid_range[0]
-#         end_date = get_invalid_range[1]
-#         status, ids = connect.add_vectors(table, vectors)
-#         status = connect.delete_vectors_by_range(table, start_date, end_date)
-#         assert not status.OK()
-
-#     """
-#     generate valid query range params, no search result
-#     """
-#     @pytest.fixture(
-#         scope="function",
-#         params=[
-#             (get_last_day(2), get_last_day(1)),
-#             (get_last_day(2), get_current_day()),
-#             (get_next_day(1), get_next_day(2))
-#         ]
-#     )
-#     def get_valid_range_no_result(self, request):
-#         yield request.param
-
-#     @pytest.mark.timeout(DELETE_TIMEOUT)
-#     def test_delete_vectors_valid_range_no_result(self, connect, table, get_valid_range_no_result):
-#         '''
-#         target: test delete vectors, no index created
-#         method: call `delete_vectors_by_range`, with valid date params
-#         expected: return code 0
-#         '''
-#         start_date = get_valid_range_no_result[0]
-#         end_date = get_valid_range_no_result[1]
-#         status, ids = connect.add_vectors(table, vectors)
-#         time.sleep(2)
-#         status = connect.delete_vectors_by_range(table, start_date, end_date)
-#         assert status.OK()
-#         status, result = connect.get_table_row_count(table)
-#         assert result == 100
-
-#     """
-#     generate valid query range params, no search result
-#     """
-#     @pytest.fixture(
-#         scope="function",
-#         params=[
-#             (get_last_day(2), get_next_day(2)),
-#             (get_current_day(), get_next_day(2)),
-#         ]
-#     )
-#     def get_valid_range(self, request):
-#         yield request.param
-
-#     @pytest.mark.timeout(DELETE_TIMEOUT)
-#     def test_delete_vectors_valid_range(self, connect, table, get_valid_range):
-#         '''
-#         target: test delete vectors, no index created
-#         method: call `delete_vectors_by_range`, with valid date params
-#         expected: return code 0
-#         '''
-#         start_date = get_valid_range[0]
-#         end_date = get_valid_range[1]
-#         status, ids = connect.add_vectors(table, vectors)
-#         time.sleep(2)
-#         status = connect.delete_vectors_by_range(table, start_date, end_date)
-#         assert status.OK()
-#         status, result = connect.get_table_row_count(table)
-#         assert result == 0
-
-#     @pytest.fixture(
-#         scope="function",
-#         params=gen_index_params()
-#     )
-#     def get_index_params(self, request, args):
-#         if "internal" not in args:
-#             if request.param["index_type"] == IndexType.IVF_SQ8H:
-#                 pytest.skip("sq8h not support in open source")
-#         return request.param
-
-#     @pytest.mark.timeout(DELETE_TIMEOUT)
-#     def test_delete_vectors_valid_range_index_created(self, connect, table, get_index_params):
-#         '''
-#         target: test delete vectors, no index created
-#         method: call `delete_vectors_by_range`, with valid date params
-#         expected: return code 0
-#         '''
-#         start_date = get_current_day()
-#         end_date = get_next_day(2)
-#         index_params = get_index_params
-#         logging.getLogger().info(index_params)
-#         status, ids = connect.add_vectors(table, vectors)
-#         status = connect.create_index(table, index_params)
-#         logging.getLogger().info(status)
-#         logging.getLogger().info("Start delete vectors by range: %s:%s" % (start_date, end_date))
-#         status = connect.delete_vectors_by_range(table, start_date, end_date)
-#         assert status.OK()
-#         status, result = connect.get_table_row_count(table)
-#         assert result == 0
-
-#     @pytest.mark.timeout(DELETE_TIMEOUT)
-#     def test_delete_vectors_no_data(self, connect, table):
-#         '''
-#         target: test delete vectors, no index created
-#         method: call `delete_vectors_by_range`, with valid date params, and no data in db
-#         expected: return code 0
-#         '''
-#         start_date = get_current_day()
-#         end_date = get_next_day(2)
-#         # status, ids = connect.add_vectors(table, vectors)
-#         status = connect.delete_vectors_by_range(table, start_date, end_date)
-#         assert status.OK()
-
-#     @pytest.mark.timeout(DELETE_TIMEOUT)
-#     def test_delete_vectors_table_not_existed(self, connect):
-#         '''
-#         target: test delete vectors, table not existed in db
-#         method: call `delete_vectors_by_range`, with table not existed
-#         expected: return code not 0
-#         '''
-#         start_date = get_current_day()
-#         end_date = get_next_day(2)
-#         table_name = gen_unique_str("not_existed_table")
-#         status = connect.delete_vectors_by_range(table_name, start_date, end_date)
-#         assert not status.OK()
-
-#     @pytest.mark.timeout(DELETE_TIMEOUT)
-#     def test_delete_vectors_table_None(self, connect, table):
-#         '''
-#         target: test delete vectors, table set Nope
-#         method: call `delete_vectors_by_range`, with table value is None
-#         expected: return code not 0
-#         '''
-#         start_date = get_current_day()
-#         end_date = get_next_day(2)
-#         table_name = None
-#         with pytest.raises(Exception) as e:
-#             status = connect.delete_vectors_by_range(table_name, start_date, end_date)
-
-#     @pytest.mark.timeout(DELETE_TIMEOUT)
-#     def test_delete_vectors_valid_range_multi_tables(self, connect, get_valid_range):
-#         '''
-#         target: test delete vectors is correct or not with multiple tables of L2
-#         method: create 50 tables and add vectors into them , then delete vectors
-#                 in valid range
-#         expected: return code 0
-#         '''
-#         nq = 100
-#         vectors = gen_vectors(nq, dim)
-#         table_list = []
-#         for i in range(50):
-#             table_name = gen_unique_str('test_delete_vectors_valid_range_multi_tables')
-#             table_list.append(table_name)
-#             param = {'table_name': table_name,
-#                      'dimension': dim,
-#                      'index_file_size': index_file_size,
-#                      'metric_type': MetricType.L2}
-#             connect.create_table(param)
-#             status, ids = connect.add_vectors(table_name=table_name, records=vectors)
-#         time.sleep(2)
-#         start_date = get_valid_range[0]
-#         end_date = get_valid_range[1]
-#         for i in range(50):
-#             status = connect.delete_vectors_by_range(table_list[i], start_date, end_date)
-#             assert status.OK()
-#             status, result = connect.get_table_row_count(table_list[i])
-#             assert result == 0
-
-
-# class TestDeleteVectorsIP:
-#     """
-#     generate invalid query range params
-#     """
-#     @pytest.fixture(
-#         scope="function",
-#         params=[
-#             (get_current_day(), get_current_day()),
-#             (get_last_day(1), get_last_day(1)),
-#             (get_next_day(1), get_next_day(1))
-#         ]
-#     )
-#     def get_invalid_range(self, request):
-#         yield request.param
-
-#     @pytest.mark.timeout(DELETE_TIMEOUT)
-#     def test_delete_vectors_invalid_range(self, connect, ip_table, get_invalid_range):
-#         '''
-#         target: test delete vectors, no index created
-#         method: call `delete_vectors_by_range`, with invalid date params
-#         expected: return code 0
-#         '''
-#         start_date = get_invalid_range[0]
-#         end_date = get_invalid_range[1]
-#         status, ids = connect.add_vectors(ip_table, vectors)
-#         status = connect.delete_vectors_by_range(ip_table, start_date, end_date)
-#         assert not status.OK()
-
-#     """
-#     generate valid query range params, no search result
-#     """
-#     @pytest.fixture(
-#         scope="function",
-#         params=[
-#             (get_last_day(2), get_last_day(1)),
-#             (get_last_day(2), get_current_day()),
-#             (get_next_day(1), get_next_day(2))
-#         ]
-#     )
-#     def get_valid_range_no_result(self, request):
-#         yield request.param
-
-#     @pytest.mark.timeout(DELETE_TIMEOUT)
-#     def test_delete_vectors_valid_range_no_result(self, connect, ip_table, get_valid_range_no_result):
-#         '''
-#         target: test delete vectors, no index created
-#         method: call `delete_vectors_by_range`, with valid date params
-#         expected: return code 0
-#         '''
-#         start_date = get_valid_range_no_result[0]
-#         end_date = get_valid_range_no_result[1]
-#         status, ids = connect.add_vectors(ip_table, vectors)
-#         time.sleep(2)
-#         status = connect.delete_vectors_by_range(ip_table, start_date, end_date)
-#         assert status.OK()
-#         status, result = connect.get_table_row_count(ip_table)
-#         assert result == 100
-
-#     """
-#     generate valid query range params, no search result
-#     """
-#     @pytest.fixture(
-#         scope="function",
-#         params=[
-#             (get_last_day(2), get_next_day(2)),
-#             (get_current_day(), get_next_day(2)),
-#         ]
-#     )
-#     def get_valid_range(self, request):
-#         yield request.param
-
-#     @pytest.mark.timeout(DELETE_TIMEOUT)
-#     def test_delete_vectors_valid_range(self, connect, ip_table, get_valid_range):
-#         '''
-#         target: test delete vectors, no index created
-#         method: call `delete_vectors_by_range`, with valid date params
-#         expected: return code 0
-#         '''
-#         start_date = get_valid_range[0]
-#         end_date = get_valid_range[1]
-#         status, ids = connect.add_vectors(ip_table, vectors)
-#         time.sleep(2)
-#         status = connect.delete_vectors_by_range(ip_table, start_date, end_date)
-#         assert status.OK()
-#         status, result = connect.get_table_row_count(ip_table)
-#         assert result == 0
-
-#     @pytest.fixture(
-#         scope="function",
-#         params=gen_index_params()
-#     )
-#     def get_index_params(self, request, args):
-#         if "internal" not in args:
-#             if request.param["index_type"] == IndexType.IVF_SQ8H:
-#                 pytest.skip("sq8h not support in open source")
-#         return request.param
-
-#     @pytest.mark.timeout(DELETE_TIMEOUT)
-#     def test_delete_vectors_valid_range_index_created(self, connect, ip_table, get_index_params):
-#         '''
-#         target: test delete vectors, no index created
-#         method: call `delete_vectors_by_range`, with valid date params
-#         expected: return code 0
-#         '''
-#         start_date = get_current_day()
-#         end_date = get_next_day(2)
-#         index_params = get_index_params
-#         logging.getLogger().info(index_params)
-#         status, ids = connect.add_vectors(ip_table, vectors)
-#         status = connect.create_index(ip_table, index_params)
-#         logging.getLogger().info(status)
-#         logging.getLogger().info("Start delete vectors by range: %s:%s" % (start_date, end_date))
-#         status = connect.delete_vectors_by_range(ip_table, start_date, end_date)
-#         assert status.OK()
-#         status, result = connect.get_table_row_count(ip_table)
-#         assert result == 0
-
-#     @pytest.mark.timeout(DELETE_TIMEOUT)
-#     def test_delete_vectors_no_data(self, connect, ip_table):
-#         '''
-#         target: test delete vectors, no index created
-#         method: call `delete_vectors_by_range`, with valid date params, and no data in db
-#         expected: return code 0
-#         '''
-#         start_date = get_current_day()
-#         end_date = get_next_day(2)
-#         # status, ids = connect.add_vectors(table, vectors)
-#         status = connect.delete_vectors_by_range(ip_table, start_date, end_date)
-#         assert status.OK()
-
-#     @pytest.mark.timeout(DELETE_TIMEOUT)
-#     def test_delete_vectors_table_None(self, connect, ip_table):
-#         '''
-#         target: test delete vectors, table set Nope
-#         method: call `delete_vectors_by_range`, with table value is None
-#         expected: return code not 0
-#         '''
-#         start_date = get_current_day()
-#         end_date = get_next_day(2)
-#         table_name = None
-#         with pytest.raises(Exception) as e:
-#             status = connect.delete_vectors_by_range(table_name, start_date, end_date)
-
-#     @pytest.mark.timeout(DELETE_TIMEOUT)
-#     def test_delete_vectors_valid_range_multi_tables(self, connect, get_valid_range):
-#         '''
-#         target: test delete vectors is correct or not with multiple tables of IP
-#         method: create 50 tables and add vectors into them , then delete vectors
-#                 in valid range
-#         expected: return code 0
-#         '''
-#         nq = 100
-#         vectors = gen_vectors(nq, dim)
-#         table_list = []
-#         for i in range(50):
-#             table_name = gen_unique_str('test_delete_vectors_valid_range_multi_tables')
-#             table_list.append(table_name)
-#             param = {'table_name': table_name,
-#                      'dimension': dim,
-#                      'index_file_size': index_file_size,
-#                      'metric_type': MetricType.IP}
-#             connect.create_table(param)
-#             status, ids = connect.add_vectors(table_name=table_name, records=vectors)
-#         time.sleep(2)
-#         start_date = get_valid_range[0]
-#         end_date = get_valid_range[1]
-#         for i in range(50):
-#             status = connect.delete_vectors_by_range(table_list[i], start_date, end_date)
-#             assert status.OK()
-#             status, result = connect.get_table_row_count(table_list[i])
-#             assert result == 0
-
-# class TestDeleteVectorsParamsInvalid:
-
-#     """
-#     Test search table with invalid table names
-#     """
-#     @pytest.fixture(
-#         scope="function",
-#         params=gen_invalid_table_names()
-#     )
-#     def get_table_name(self, request):
-#         yield request.param
-
-#     @pytest.mark.level(2)
-#     def test_delete_vectors_table_invalid_name(self, connect, get_table_name):
-#         '''
-#         '''
-#         start_date = get_current_day()
-#         end_date = get_next_day(2)
-#         table_name = get_table_name
-#         logging.getLogger().info(table_name)
-#         top_k = 1
-#         nprobe = 1 
-#         status = connect.delete_vectors_by_range(table_name, start_date, end_date)
-#         assert not status.OK()
-
-#     """
-#     Test search table with invalid query ranges
-#     """
-#     @pytest.fixture(
-#         scope="function",
-#         params=gen_invalid_query_ranges()
-#     )
-#     def get_query_ranges(self, request):
-#         yield request.param
-
-#     @pytest.mark.timeout(DELETE_TIMEOUT)
-#     def test_delete_vectors_range_invalid(self, connect, table, get_query_ranges):
-#         '''
-#         target: test search fuction, with the wrong query_range
-#         method: search with query_range
-#         expected: raise an error, and the connection is normal
-#         '''
-#         start_date = get_query_ranges[0][0]
-#         end_date = get_query_ranges[0][1]
-#         status, ids = connect.add_vectors(table, vectors)
-#         logging.getLogger().info(get_query_ranges)
-#         with pytest.raises(Exception) as e:
-#             status = connect.delete_vectors_by_range(table, start_date, end_date)
\ No newline at end of file
diff --git a/tests/milvus_python_test/test_index.py b/tests/milvus_python_test/test_index.py
index 9e09b59d..cf03f856 100644
--- a/tests/milvus_python_test/test_index.py
+++ b/tests/milvus_python_test/test_index.py
@@ -9,7 +9,7 @@ import threading
 from multiprocessing import Pool, Process
 import numpy
 import sklearn.preprocessing
-from milvus import Milvus, IndexType, MetricType
+from milvus import IndexType, MetricType
 from utils import *
 
 nb = 6000
@@ -146,7 +146,7 @@ class TestIndexBase:
         uri = "tcp://%s:%s" % (args["ip"], args["port"])
 
         for i in range(process_num):
-            m = Milvus()
+            m = get_milvus()
             m.connect(uri=uri)
             p = Process(target=build, args=(m,))
             processes.append(p)
@@ -205,7 +205,7 @@ class TestIndexBase:
         uri = "tcp://%s:%s" % (args["ip"], args["port"])
 
         for i in range(process_num):
-            m = Milvus()
+            m = get_milvus()
             m.connect(uri=uri)
             ids = i
             p = Process(target=create_index, args=(m,ids))
@@ -669,7 +669,7 @@ class TestIndexIP:
         uri = "tcp://%s:%s" % (args["ip"], args["port"])
 
         for i in range(process_num):
-            m = Milvus()
+            m = get_milvus()
             m.connect(uri=uri)
             p = Process(target=build, args=(m,))
             processes.append(p)
@@ -726,7 +726,7 @@ class TestIndexIP:
         uri = "tcp://%s:%s" % (args["ip"], args["port"])
 
         for i in range(process_num):
-            m = Milvus()
+            m = get_milvus()
             m.connect(uri=uri)
             ids = i
             p = Process(target=create_index, args=(m,ids))
diff --git a/tests/milvus_python_test/test_mix.py b/tests/milvus_python_test/test_mix.py
index d9331151..f7baa8dd 100644
--- a/tests/milvus_python_test/test_mix.py
+++ b/tests/milvus_python_test/test_mix.py
@@ -7,7 +7,7 @@ import logging
 from time import sleep
 from multiprocessing import Process
 import sklearn.preprocessing
-from milvus import Milvus, IndexType, MetricType
+from milvus import IndexType, MetricType
 from utils import *
 
 dim = 128
@@ -32,7 +32,7 @@ class TestMixBase:
         query_vecs = [vectors[0], vectors[1]]
         uri = "tcp://%s:%s" % (args["ip"], args["port"])
         id_0 = 0; id_1 = 0
-        milvus_instance = Milvus()
+        milvus_instance = get_milvus()
         milvus_instance.connect(uri=uri)
         milvus_instance.create_table({'table_name': table,
              'dimension': dim,
@@ -60,11 +60,11 @@ class TestMixBase:
                 logging.getLogger().info(status)
                 assert result[0][0].id == id_0
                 assert result[1][0].id == id_1
-        milvus_instance = Milvus()
+        milvus_instance = get_milvus()
         milvus_instance.connect(uri=uri)
         p_search = Process(target=search, args=(milvus_instance, ))
         p_search.start()
-        milvus_instance = Milvus()
+        milvus_instance = get_milvus()
         milvus_instance.connect(uri=uri)
         p_create = Process(target=add_vectors, args=(milvus_instance, ))
         p_create.start()
diff --git a/tests/milvus_python_test/test_partition.py b/tests/milvus_python_test/test_partition.py
index cbb0b5bc..cdf492e4 100644
--- a/tests/milvus_python_test/test_partition.py
+++ b/tests/milvus_python_test/test_partition.py
@@ -5,7 +5,7 @@ import threading
 import logging
 from multiprocessing import Pool, Process
 import pytest
-from milvus import Milvus, IndexType, MetricType
+from milvus import IndexType, MetricType
 from utils import *
 
 
diff --git a/tests/milvus_python_test/test_search_vectors.py b/tests/milvus_python_test/test_search_vectors.py
index 1f015f41..237e4521 100644
--- a/tests/milvus_python_test/test_search_vectors.py
+++ b/tests/milvus_python_test/test_search_vectors.py
@@ -9,7 +9,7 @@ import logging
 from time import sleep
 from multiprocessing import Process
 import numpy
-from milvus import Milvus, IndexType, MetricType
+from milvus import IndexType, MetricType
 from utils import *
 
 dim = 128
@@ -743,7 +743,7 @@ class TestSearchBase:
              'index_type': IndexType.FLAT,
              'store_raw_vector': False}
         # create table
-        milvus = Milvus()
+        milvus = get_milvus()
         milvus.connect(uri=uri)
         milvus.create_table(param)
         vectors, ids = self.init_data(milvus, table, nb=nb)
@@ -756,7 +756,7 @@ class TestSearchBase:
                 assert result[i][0].distance == 0.0
 
         for i in range(process_num):
-            milvus = Milvus()
+            milvus = get_milvus()
             milvus.connect(uri=uri)
             p = Process(target=search, args=(milvus, ))
             processes.append(p)
@@ -784,7 +784,7 @@ class TestSearchBase:
                      'index_file_size': 10,
                      'metric_type': MetricType.L2}
             # create table
-            milvus = Milvus()
+            milvus = get_milvus()
             milvus.connect(uri=uri)
             milvus.create_table(param)
             status, ids = milvus.add_vectors(table, vectors)
@@ -826,7 +826,7 @@ class TestSearchBase:
                      'index_file_size': 10,
                      'metric_type': MetricType.L2}
             # create table
-            milvus = Milvus()
+            milvus = get_milvus()
             milvus.connect(uri=uri)
             milvus.create_table(param)
             status, ids = milvus.add_vectors(table, vectors)
diff --git a/tests/milvus_python_test/test_table.py b/tests/milvus_python_test/test_table.py
index 4f03a950..84c333f5 100644
--- a/tests/milvus_python_test/test_table.py
+++ b/tests/milvus_python_test/test_table.py
@@ -3,11 +3,9 @@ import pdb
 import pytest
 import logging
 import itertools
-
+import numpy
 from time import sleep
 from multiprocessing import Process
-import numpy
-from milvus import Milvus
 from milvus import IndexType, MetricType
 from utils import *
 
@@ -280,7 +278,7 @@ class TestTable:
         process_num = 4
         processes = []
         for i in range(process_num):
-            milvus = Milvus()
+            milvus = get_milvus()
             milvus.connect(uri=uri)
             p = Process(target=describetable, args=(milvus,))
             processes.append(p)
@@ -460,7 +458,7 @@ class TestTable:
             assert status.OK()
 
         for i in range(process_num):
-            milvus = Milvus()
+            milvus = get_milvus()
             milvus.connect(uri=uri)
             p = Process(target=deletetable, args=(milvus,))
             processes.append(p)
@@ -713,7 +711,7 @@ class TestTable:
         processes = []
 
         for i in range(process_num):
-            milvus = Milvus()
+            milvus = get_milvus()
             milvus.connect(uri=uri)
             p = Process(target=showtables, args=(milvus,))
             processes.append(p)
diff --git a/tests/milvus_python_test/test_table_count.py b/tests/milvus_python_test/test_table_count.py
index 4ef73472..feb2156e 100644
--- a/tests/milvus_python_test/test_table_count.py
+++ b/tests/milvus_python_test/test_table_count.py
@@ -1,15 +1,12 @@
 import random
 import pdb
-
 import pytest
 import logging
 import itertools
-
 from time import sleep
 from multiprocessing import Process
-from milvus import Milvus
-from utils import *
 from milvus import IndexType, MetricType
+from utils import *
 
 dim = 128
 index_file_size = 10
@@ -209,7 +206,7 @@ class TestTableCount:
         process_num = 8
         processes = []
         for i in range(process_num):
-            milvus = Milvus()
+            milvus = get_milvus()
             milvus.connect(uri=uri)
             p = Process(target=rows_count, args=(milvus, ))
             processes.append(p)
@@ -354,7 +351,7 @@ class TestTableCountIP:
         process_num = 8
         processes = []
         for i in range(process_num):
-            milvus = Milvus()
+            milvus = get_milvus()
             milvus.connect(uri=uri)
             p = Process(target=rows_count, args=(milvus,))
             processes.append(p)
diff --git a/tests/milvus_python_test/utils.py b/tests/milvus_python_test/utils.py
index 7be98a36..c1115155 100644
--- a/tests/milvus_python_test/utils.py
+++ b/tests/milvus_python_test/utils.py
@@ -7,9 +7,16 @@ import logging
 import time, datetime
 import copy
 import numpy as np
-from utils import *
 from milvus import Milvus, IndexType, MetricType
 
+port = 19530
+
+
+def get_milvus(handler=None):
+    if handler is None:
+        handler = "GRPC"
+    return Milvus(handler=handler)
+
 
 def gen_inaccuracy(num):
     return num/255.0
@@ -125,12 +132,9 @@ def gen_invalid_ports():
 
 def gen_invalid_uris():
     ip = None
-    port = 19530
-
     uris = [
             " ",
             "中文",
-
             # invalid protocol
             # "tc://%s:%s" % (ip, port),
             # "tcp%s:%s" % (ip, port),
@@ -143,15 +147,14 @@ def gen_invalid_uris():
             # "tcp://%s:string" % ip,
 
             # invalid ip
-            "tcp:// :%s" % port,
+            "tcp:// :19530",
             # "tcp://123.0.0.1:%s" % port,
-            "tcp://127.0.0:%s" % port,
+            "tcp://127.0.0:19530",
             # "tcp://255.0.0.0:%s" % port,
             # "tcp://255.255.0.0:%s" % port,
             # "tcp://255.255.255.0:%s" % port,
             # "tcp://255.255.255.255:%s" % port,
-            "tcp://\n:%s" % port,
-
+            "tcp://\n:19530",
     ]
     return uris
 
-- 
GitLab