add refactor test codes (#5171)

Signed-off-by: Nwangting0128 <ting.wang@zilliz.com>

tests20/python_client/README.md

-## python_client
-
-### Test levels
-#### L0
-#### L1
-#### L2
-
-### Run a test
-
-### Contribute a test
+# refactor_test
\ No newline at end of file

tests20/python_client/base/ClientRequest.py

+import pytest
+import sys
+
+sys.path.append("..")
+from base.Connections import ApiConnections
+from base.Collection import ApiCollection
+from base.Partition import ApiPartition
+from base.Index import ApiIndex
+from base.Utility import ApiUtility
+
+from config.my_info import my_info
+from common.common_func import *
+from check.func_check import *
+
+
+def request_catch():
+    def wrapper(func):
+        def inner_wrapper(*args, **kwargs):
+            try:
+                return func(*args, **kwargs), True
+            except Exception as e:
+                log.error("[ClientRequest API Exception]%s: %s" % (str(func), str(e)))
+                return e, False
+        return inner_wrapper
+    return wrapper
+
+
+@request_catch()
+def func_req(_list, **kwargs):
+    if isinstance(_list, list):
+        func = _list[0]
+        if callable(func):
+            arg = []
+            if len(_list) > 1:
+                for a in _list[1:]:
+                    arg.append(a)
+            return func(*arg, **kwargs)
+    return False, False
+
+
+class ParamInfo:
+    def __init__(self):
+        self.param_ip = ""
+        self.param_port = ""
+        self.param_handler = ""
+
+    def prepare_param_info(self, ip, port, handler):
+        self.param_ip = ip
+        self.param_port = port
+        self.param_handler = handler
+
+
+param_info = ParamInfo()
+
+
+class Base:
+
+    """ Initialize class object """
+    connection = None
+    collection = None
+    partition = None
+    index = None
+    utility = None
+
+    def setup_class(self):
+        log.info("[setup_class] Start setup class...")
+
+    def teardown_class(self):
+        pass
+
+    def setup(self):
+        log.error("*" * 80)
+        self.connection = ApiConnections()
+        self.collection = ApiCollection()
+        self.partition = ApiPartition()
+        self.index = ApiIndex()
+        self.utility = ApiUtility()
+
+    def teardown(self):
+        pass
+
+    @pytest.fixture(scope="module", autouse=True)
+    def initialize_env(self, request):
+        """ clean log before testing """
+        modify_file([my_info.test_log, my_info.test_err])
+        log.info("[initialize_milvus] Log cleaned up, start testing...")
+
+        ip = request.config.getoption("--ip")
+        port = request.config.getoption("--port")
+        handler = request.config.getoption("--handler")
+        param_info.prepare_param_info(ip, port, handler)
+
+
+class ApiReq(Base):
+    """
+    Additional methods;
+    Public methods that can be used to add cases.
+    """
+
+    def func(self):
+        pass
+
+    @staticmethod
+    def func_2():
+        pass

tests20/python_client/base/Collection.py

+from pymilvus_orm import Collection
+from pymilvus_orm.types import DataType
+from pymilvus_orm.default_config import DefaultConfig
+import sys
+
+sys.path.append("..")
+from check.param_check import *
+from check.func_check import *
+from utils.util_log import my_log as log
+from common.common_type import *
+
+
+def collection_catch():
+    def wrapper(func):
+        def inner_wrapper(*args, **kwargs):
+            try:
+                return func(*args, **kwargs), True
+            except Exception as e:
+                log.error("[Collection API Exception]%s: %s" % (str(func), str(e)))
+                return e, False
+        return inner_wrapper
+    return wrapper
+
+
+@collection_catch()
+def func_req(_list, **kwargs):
+    if isinstance(_list, list):
+        func = _list[0]
+        if callable(func):
+            arg = []
+            if len(_list) > 1:
+                for a in _list[1:]:
+                    arg.append(a)
+            return func(*arg, **kwargs)
+    return False, False
+
+
+class ApiCollection:
+    collection = None
+
+    def collection_init(self, name, data=None, schema=None, check_res=None, **kwargs):
+        """ In order to distinguish the same name of collection """
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([Collection, name, data, schema], **kwargs)
+        self.collection = res if check is True else None
+        check_result = CheckFunc(res, func_name, check_res, name=name, data=data, schema=schema, **kwargs).run()
+        return res, check_result
+
+    def schema(self, check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.collection.schema])
+        check_result = CheckFunc(res, func_name, check_res).run()
+        return res, check_result
+
+    def description(self, check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.collection.description])
+        check_result = CheckFunc(res, func_name, check_res).run()
+        return res, check_result
+
+    def name(self, check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.collection.name])
+        check_result = CheckFunc(res, func_name, check_res).run()
+        return res, check_result
+
+    def is_empty(self, check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.collection.is_empty])
+        check_result = CheckFunc(res, func_name, check_res).run()
+        return res, check_result
+
+    def num_entities(self, check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.collection.num_entities])
+        check_result = CheckFunc(res, func_name, check_res).run()
+        return res, check_result
+
+    def primary_field(self, check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.collection.primary_field])
+        check_result = CheckFunc(res, func_name, check_res).run()
+        return res, check_result
+
+    def drop(self, check_res=None, **kwargs):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.collection.drop], **kwargs)
+        check_result = CheckFunc(res, func_name, check_res, **kwargs).run()
+        return res, check_result
+
+    def load(self, field_names=None, index_names=None, partition_names=None, check_res=None, **kwargs):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.collection.load, field_names, index_names, partition_names], **kwargs)
+        check_result = CheckFunc(res, func_name, check_res, field_names=field_names, index_names=index_names,
+                                 partition_names=partition_names, **kwargs).run()
+        return res, check_result
+
+    def release(self, check_res=None, **kwargs):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.collection.release], **kwargs)
+        check_result = CheckFunc(res, func_name, check_res, **kwargs).run()
+        return res, check_result
+
+    def insert(self, data, partition_name=None, check_res=None, **kwargs):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.collection.insert, data, partition_name], **kwargs)
+        check_result = CheckFunc(res, func_name, check_res, dat=data, partition_name=partition_name, **kwargs).run()
+        return res, check_result
+
+    def search(self, data, anns_field, param, limit, expression, partition_names=None, output_fields=None, timeout=None,
+               check_res=None, **kwargs):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.collection.search, data, anns_field, param, limit, expression, partition_names,
+                               output_fields, timeout], **kwargs)
+        check_result = CheckFunc(res, func_name, check_res, data=data, anns_field=anns_field, param=param, limit=limit,
+                                 expression=expression, partition_names=partition_names, output_fields=output_fields,
+                                 timeout=timeout, **kwargs).run()
+        return res, check_result
+
+    def partitions(self, check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.collection.partitions])
+        check_result = CheckFunc(res, func_name, check_res).run()
+        return res, check_result
+
+    def partition(self, partition_name, check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.collection.partition, partition_name])
+        check_result = CheckFunc(res, func_name, check_res, partition_name=partition_name).run()
+        return res, check_result
+
+    def has_partition(self, partition_name, check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.collection.has_partition, partition_name])
+        check_result = CheckFunc(res, func_name, check_res, partition_name=partition_name).run()
+        return res, check_result
+
+    def drop_partition(self, partition_name, check_res=None, **kwargs):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.collection.drop_partition, partition_name], **kwargs)
+        check_result = CheckFunc(res, func_name, check_res, partition_name=partition_name, **kwargs).run()
+        return res, check_result
+
+    def indexes(self, check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.collection.indexes])
+        check_result = CheckFunc(res, func_name, check_res).run()
+        return res, check_result
+
+    def index(self, index_name="", check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.collection.index, index_name])
+        check_result = CheckFunc(res, func_name, check_res, index_name=index_name).run()
+        return res, check_result
+
+    def create_index(self, field_name, index_params, index_name="", check_res=None, **kwargs):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.collection.create_index, field_name, index_params, index_name], **kwargs)
+        check_result = CheckFunc(res, func_name, check_res, field_name=field_name, index_params=index_params,
+                                 index_name=index_name, **kwargs).run()
+        return res, check_result
+
+    def has_index(self, index_name="", check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.collection.has_index, index_name])
+        check_result = CheckFunc(res, func_name, check_res, index_name=index_name).run()
+        return res, check_result
+
+    def drop_index(self, index_name="", check_res=None, **kwargs):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.collection.drop_index, index_name], **kwargs)
+        check_result = CheckFunc(res, func_name, check_res, index_name=index_name, **kwargs).run()
+        return res, check_result

tests20/python_client/base/Connections.py

+from pymilvus_orm import Connections
+from pymilvus_orm.types import DataType
+from pymilvus_orm.default_config import DefaultConfig
+import sys
+
+sys.path.append("..")
+from check.param_check import *
+from check.func_check import *
+from utils.util_log import my_log as log
+from common.common_type import *
+
+
+def connections_catch():
+    def wrapper(func):
+        def inner_wrapper(*args, **kwargs):
+            try:
+                return func(*args, **kwargs), True
+            except Exception as e:
+                log.error("[Connections API Exception]%s: %s" % (str(func), str(e)))
+                return e, False
+        return inner_wrapper
+    return wrapper
+
+
+@connections_catch()
+def func_req(_list, **kwargs):
+    if isinstance(_list, list):
+        func = _list[0]
+        if callable(func):
+            arg = []
+            if len(_list) > 1:
+                for a in _list[1:]:
+                    arg.append(a)
+            return func(*arg, **kwargs)
+    return False, False
+
+
+class ApiConnections:
+    def __init__(self):
+        self.connection = Connections()
+
+    def configure(self, check_res=None, **kwargs):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.connection.configure], **kwargs)
+        check_result = CheckFunc(res, func_name, check_res, **kwargs).run()
+        return res, check_result
+
+    def remove_connection(self, alias, check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.connection.remove_connection, alias])
+        check_result = CheckFunc(res, func_name, check_res, alias=alias).run()
+        return res, check_result
+
+    def create_connection(self, alias=DefaultConfig.DEFAULT_USING, check_res=None, **kwargs):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.connection.create_connection, alias], **kwargs)
+        check_result = CheckFunc(res, func_name, check_res, alias=alias, **kwargs).run()
+        return res, check_result
+
+    def get_connection(self, alias=DefaultConfig.DEFAULT_USING, check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.connection.get_connection, alias])
+        check_result = CheckFunc(res, func_name, check_res, alias=alias).run()
+        return res, check_result
+
+    def list_connections(self, check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.connection.list_connections])
+        check_result = CheckFunc(res, func_name, check_res).run()
+        return res, check_result
+
+    def get_connection_addr(self, alias, check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.connection.get_connection_addr, alias])
+        check_result = CheckFunc(res, func_name, check_res, alias=alias).run()
+        return res, check_result

tests20/python_client/base/Index.py

+from pymilvus_orm import Index
+from pymilvus_orm.types import DataType
+from pymilvus_orm.default_config import DefaultConfig
+import sys
+
+sys.path.append("..")
+from check.param_check import *
+from check.func_check import *
+from utils.util_log import my_log as log
+from common.common_type import *
+
+
+def index_catch():
+    def wrapper(func):
+        def inner_wrapper(*args, **kwargs):
+            try:
+                return func(*args, **kwargs), True
+            except Exception as e:
+                log.error("[Index API Exception]%s: %s" % (str(func), str(e)))
+                return e, False
+        return inner_wrapper
+    return wrapper
+
+
+@index_catch()
+def func_req(_list, **kwargs):
+    if isinstance(_list, list):
+        func = _list[0]
+        if callable(func):
+            arg = []
+            if len(_list) > 1:
+                for a in _list[1:]:
+                    arg.append(a)
+            return func(*arg, **kwargs)
+    return False, False
+
+
+class ApiIndex:
+    index = None
+
+    def index_init(self, collection, field_name, index_params, name="", check_res=None, **kwargs):
+        """ In order to distinguish the same name of index """
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([Index, collection, field_name, index_params, name], **kwargs)
+        self.index = res if check is True else None
+        check_result = CheckFunc(res, func_name, check_res, collection=collection, field_name=field_name,
+                                 index_params=index_params, name=name, **kwargs).run()
+        return res, check_result
+
+    def name(self, check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.index.name])
+        check_result = CheckFunc(res, func_name, check_res).run()
+        return res, check_result
+
+    def params(self, check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.index.params])
+        check_result = CheckFunc(res, func_name, check_res).run()
+        return res, check_result
+
+    def collection_name(self, check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.index.collection_name])
+        check_result = CheckFunc(res, func_name, check_res).run()
+        return res, check_result
+
+    def field_name(self, check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.index.field_name])
+        check_result = CheckFunc(res, func_name, check_res).run()
+        return res, check_result
+
+    def drop(self, check_res=None, **kwargs):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.index.drop], **kwargs)
+        check_result = CheckFunc(res, func_name, check_res, **kwargs).run()
+        return res, check_result

tests20/python_client/base/Partition.py

+from pymilvus_orm import Partition
+from pymilvus_orm.types import DataType
+from pymilvus_orm.default_config import DefaultConfig
+import sys
+
+sys.path.append("..")
+from check.param_check import *
+from check.func_check import *
+from utils.util_log import my_log as log
+from common.common_type import *
+
+
+def partition_catch():
+    def wrapper(func):
+        def inner_wrapper(*args, **kwargs):
+            try:
+                return func(*args, **kwargs), True
+            except Exception as e:
+                log.error("[Partition API Exception]%s: %s" % (str(func), str(e)))
+                return e, False
+        return inner_wrapper
+    return wrapper
+
+
+@partition_catch()
+def func_req(_list, **kwargs):
+    if isinstance(_list, list):
+        func = _list[0]
+        if callable(func):
+            arg = []
+            if len(_list) > 1:
+                for a in _list[1:]:
+                    arg.append(a)
+            return func(*arg, **kwargs)
+    return False, False
+
+
+class ApiPartition:
+    partition = None
+
+    def partition_init(self, collection, name, description="", check_res=None, **kwargs):
+        """ In order to distinguish the same name of partition """
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([Partition, collection, name, description], **kwargs)
+        self.partition = res if check is True else None
+        check_result = CheckFunc(res, func_name, check_res, collection=collection, name=name, description=description,
+                                 **kwargs).run()
+        return res, check_result
+
+    def description(self, check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.partition.description])
+        check_result = CheckFunc(res, func_name, check_res).run()
+        return res, check_result
+
+    def name(self, check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.partition.name])
+        check_result = CheckFunc(res, func_name, check_res).run()
+        return res, check_result
+
+    def is_empty(self, check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.partition.is_empty])
+        check_result = CheckFunc(res, func_name, check_res).run()
+        return res, check_result
+
+    def num_entities(self, check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.partition.num_entities])
+        check_result = CheckFunc(res, func_name, check_res).run()
+        return res, check_result
+
+    def drop(self, check_res=None, **kwargs):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.partition.drop], **kwargs)
+        check_result = CheckFunc(res, func_name, check_res, **kwargs).run()
+        return res, check_result
+
+    def load(self, field_names=None, index_names=None, check_res=None, **kwargs):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.partition.load, field_names, index_names], **kwargs)
+        check_result = CheckFunc(res, func_name, check_res, field_names=field_names, index_names=index_names,
+                                 **kwargs).run()
+        return res, check_result
+
+    def release(self, check_res=None, **kwargs):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.partition.release], **kwargs)
+        check_result = CheckFunc(res, func_name, check_res, **kwargs).run()
+        return res, check_result
+
+    def insert(self, data, check_res=None, **kwargs):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.partition.insert, data], **kwargs)
+        check_result = CheckFunc(res, func_name, check_res, data=data, **kwargs).run()
+        return res, check_result
+
+    def search(self, data, anns_field, params, limit, expr=None, output_fields=None, check_res=None, **kwargs):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.partition.search, data, anns_field, params, limit, expr, output_fields], **kwargs)
+        check_result = CheckFunc(res, func_name, check_res, data=data, anns_field=anns_field, params=params,
+                                 limit=limit, expr=expr, output_fields=output_fields, **kwargs).run()
+        return res, check_result

tests20/python_client/base/Utility.py

+from pymilvus_orm import utility
+from pymilvus_orm.types import DataType
+from pymilvus_orm.default_config import DefaultConfig
+import sys
+
+sys.path.append("..")
+from check.param_check import *
+from check.func_check import *
+from utils.util_log import my_log as log
+from common.common_type import *
+
+
+def utility_catch():
+    def wrapper(func):
+        def inner_wrapper(*args, **kwargs):
+            try:
+                return func(*args, **kwargs), True
+            except Exception as e:
+                log.error("[Utility API Exception]%s: %s" % (str(func), str(e)))
+                return e, False
+        return inner_wrapper
+    return wrapper
+
+
+@utility_catch()
+def func_req(_list, **kwargs):
+    if isinstance(_list, list):
+        func = _list[0]
+        if callable(func):
+            arg = []
+            if len(_list) > 1:
+                for a in _list[1:]:
+                    arg.append(a)
+            return func(*arg, **kwargs)
+    return False, False
+
+
+class ApiUtility:
+    """ Method of encapsulating utility files """
+
+    ut = utility
+
+    def loading_progress(self, collection_name, partition_names=[], using="default", check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.ut.loading_progress, collection_name, partition_names, using])
+        check_result = CheckFunc(res, func_name, check_res, collection_name=collection_name,
+                                 partition_names=partition_names,using=using).run()
+        return res, check_result
+
+    def wait_for_loading_complete(self, collection_name, partition_names=[], timeout=None, using="default", check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.ut.wait_for_loading_complete, collection_name, partition_names, timeout, using])
+        check_result = CheckFunc(res, func_name, check_res, collection_name=collection_name,
+                                 partition_names=partition_names, timeout=timeout, using=using).run()
+        return res, check_result
+
+    def index_building_progress(self, collection_name, index_name="", using="default", check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.ut.index_building_progress, collection_name, index_name, using])
+        check_result = CheckFunc(res, func_name, check_res, collection_name=collection_name, index_name=index_name,
+                                 using=using).run()
+        return res, check_result
+
+    def wait_for_index_building_complete(self, collection_name, index_name="", timeout=None, using="default", check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.ut.wait_for_loading_complete, collection_name, index_name, timeout, using])
+        check_result = CheckFunc(res, func_name, check_res, collection_name=collection_name, index_name=index_name,
+                                 timeout=timeout, using=using).run()
+        return res, check_result
+
+    def has_collection(self, collection_name, using="default", check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.ut.has_collection, collection_name, using])
+        check_result = CheckFunc(res, func_name, check_res, collection_name=collection_name, using=using).run()
+        return res, check_result
+
+    def has_partition(self, collection_name, partition_name, using="default", check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.ut.has_partition, collection_name, partition_name, using])
+        check_result = CheckFunc(res, func_name, check_res, collection_name=collection_name,
+                                 partition_name=partition_name, using=using).run()
+        return res, check_result
+
+    def list_collections(self, timeout=None, using="default", check_res=None):
+        func_name = sys._getframe().f_code.co_name
+        res, check = func_req([self.ut.list_collections, timeout, using])
+        check_result = CheckFunc(res, func_name, check_res, timeout=timeout,  using=using).run()
+        return res, check_result

tests20/python_client/check/func_check.py

+from utils.util_log import my_log as log
+from common.common_type import *
+
+
+class CheckFunc:
+    def __init__(self, res, func_name, check_res, **kwargs):
+        self.res = res  # response of api request
+        self.func_name = func_name
+        self.check_res = check_res
+        self.params = {}
+
+        for key, value in kwargs.items():
+            self.params[key] = value
+
+        self.keys = self.params.keys()
+
+    def run(self):
+        check_result = True
+
+        if self.check_res is None:
+            log.info("self.check_res is None, the response of API: %s" % self.res)
+        elif self.check_res == cname_param_check:
+            check_result = self.req_cname_check(self.res, self.func_name, self.params.get('collection_name'))
+        elif self.check_res == pname_param_check:
+            check_result = self.req_pname_check(self.res, self.func_name, self.params.get('partition_tag'))
+
+        return check_result
+
+    @staticmethod
+    def req_cname_check(res, func_name, params):
+        code = getattr(res, 'code', "The exception does not contain the field of code.")
+        message = getattr(res, 'message', "The exception does not contain the field of message.")
+
+        if not isinstance(params, str):
+            log.info("[req_params_check] Check param is not a str.")
+            if func_name in ["drop_collection", "has_collection", "describe_collection", "load_collection",
+                             "release_collection", "list_partitions", "create_partition", "drop_partition",
+                             "has_partition", "load_partitions", "release_partitions", "drop_index", "describe_index"]:
+                assert res.args[0] == "`collection_name` value %s is illegal" % str(params)
+            elif func_name in ["create_collection", "create_index", "search"]:
+                if params is None:
+                    if func_name in ["search"]:
+                        assert code == 1
+                        assert message == "collection not exists"
+                    else:
+                        assert code == 1
+                        assert message == "Collection name should not be empty"
+                else:
+                    # check_str = "Cannot set milvus.proto.schema.CollectionSchema.name to" + \
+                    #             " %s: %s has type %s," % (str(params), str(params), str(type(params))) +\
+                    #             " but expected one of: (<class 'bytes'>, <class 'str'>) for field CollectionSchema.name"
+                    # assert res.args[0] == check_str
+                    check_str = "%s has type %s, but expected one of: bytes, unicode" % (str(params), str(type(params))[8:-2])
+                    assert res.args[0] == check_str
+            # elif func_name == "create_index":
+            #     if params is None:
+            #         assert code == 1
+            #         assert message == "Collection name should not be empty"
+            #     else:
+            #         check_str = "Cannot set milvus.proto.milvus.DescribeCollectionRequest.collection_name to" + \
+            #                     " %s: %s has type %s," % (str(params), str(params), str(type(params))) +\
+            #                     " but expected one of: (<class 'bytes'>, <class 'str'>) for field DescribeCollectionRequest.collection_name"
+            #         assert res.args[0] == check_str
+            # elif func_name == "search":
+            #     if params is None:
+            #         assert code == 1
+            #         assert message == "collection not exists"
+            #     else:
+            #         check_str = "Cannot set milvus.proto.milvus.HasCollectionRequest.collection_name to" + \
+            #                     " %s: %s has type %s," % (str(params), str(params), str(type(params))) +\
+            #                     " but expected one of: (<class 'bytes'>, <class 'str'>) for field HasCollectionRequest.collection_name"
+            #         assert res.args[0] == check_str
+            elif func_name == "flush":
+                if params is None or params == []:
+                    assert res.args[0] == "Collection name list can not be None or empty"
+                elif params == [1, "2", 3]:
+                    assert res.args[0] == "`collection_name` value 1 is illegal"
+                else:
+                    assert res.args[0] == "Collection name array must be type of list"
+
+        elif isinstance(params, str) and len(params) < 256:
+            log.info("[req_params_check] Check str param less than 256.")
+            if func_name in ["create_collection", "drop_collection", "has_collection", "describe_collection",
+                             "load_collection", "release_collection", "list_partitions", "create_partition",
+                             "drop_partition", "has_partition", "load_partitions", "release_partitions", "drop_index", "describe_index"]:
+                assert code == 1
+                assert message == "Invalid collection name: %s. The first character of a collection name must be an underscore or letter." % params
+            elif func_name == "flush":
+                assert res.args[0] == "Collection name array must be type of list"
+            elif func_name == "create_index":
+                assert code == 1
+                assert message == "Invalid collection name: %s. The first character of a collection name must be an underscore or letter." % params
+            elif func_name == "search":
+                assert code == 1
+                assert message == "collection not exists"
+
+        elif isinstance(params, str) and len(params) >= 256:
+            log.info("[req_params_check] Check str param more than 256.")
+            if func_name in ["create_collection", "drop_collection", "has_collection", "describe_collection",
+                             "load_collection", "release_collection", "list_partitions", "create_partition",
+                             "drop_partition", "has_partition", "load_partitions", "release_partitions", "drop_index",
+                             "describe_index", "create_index"]:
+                assert code == 1
+                assert message == "Invalid collection name: %s. The length of a collection name must be less than 255 characters." % params
+            elif func_name == "flush":
+                assert res.args[0] == "Collection name array must be type of list"
+            elif func_name == "search":
+                assert code == 1
+                assert message == "collection not exists"
+
+        return True
+
+    @staticmethod
+    def req_pname_check(res, func_name, params):
+        code = getattr(res, 'code', "The exception does not contain the field of code.")
+        message = getattr(res, 'message', "The exception does not contain the field of message.")
+
+        if not isinstance(params, str):
+            log.info("[req_pname_check] Check param is not a str.")
+            log.info(res.args[0])
+            if func_name in ["create_partition", "drop_partition", "has_partition"]:
+                assert res.args[0] == "`partition_tag` value %s is illegal" % str(params)
+
+        elif isinstance(params, str) and len(params) < 256:
+            log.info("[req_pname_check] Check str param less than 256.")
+            if func_name in ["create_partition", "drop_partition", "has_partition"]:
+                assert code == 1
+                check_str = ["Invalid partition tag: %s. Partition tag can only contain numbers, letters, dollars and underscores." % str(params),
+                             "Invalid partition tag: %s. The first character of a partition tag must be an underscore or letter." % str(params)]
+                assert message in check_str
+
+        elif isinstance(params, str) and len(params) >= 256:
+            log.info("[req_pname_check] Check str param more than 256.")
+            if func_name in ["create_partition", "drop_partition", "has_partition"]:
+                assert code == 1
+                assert message == "Invalid partition tag: %s. The length of a partition tag must be less than 255 characters." % str(params)
+
+        return True

tests20/python_client/check/param_check.py

+import pytest
+import sys
+
+sys.path.append("..")
+from utils.util_log import my_log
+
+
+def ip_check(ip):
+    if ip == "localhost":
+        return True
+
+    if not isinstance(ip, str):
+        my_log.error("[IP_CHECK] IP(%s) is not a string." % ip)
+        return False
+
+    _list = ip.split('.')
+    if len(_list) != 4:
+        my_log.error("[IP_CHECK] IP(%s) is wrong, please check manually." % ip)
+        return False
+
+    for i in _list:
+        if not str(i).isdigit():
+            my_log.error("[IP_CHECK] IP(%s) is wrong, please check manually." % ip)
+            return False
+
+    return True
+
+
+def number_check(num):
+    if str(num).isdigit():
+        return True
+
+    else:
+        my_log.error("[NUMBER_CHECK] Number(%s) is not a numbers." % num)
+        return False
+
+
+def exist_check(param, _list):
+    if param in _list:
+        return True
+
+    else:
+        my_log.error("[EXIST_CHECK] Param(%s) is not in (%s)" % (param, _list))
+        return False

tests20/python_client/common/common_func.py

+import os
+import random
+import string
+import numpy as np
+from sklearn import preprocessing
+
+from pymilvus_orm.types import DataType
+from utils.util_log import my_log as log
+from common.common_type import *
+
+
+"""" Methods of processing data """
+l2 = lambda x, y: np.linalg.norm(np.array(x) - np.array(y))
+
+
+def get_binary_default_fields(auto_id=True):
+    default_fields = {
+        "fields": [
+            {"name": "int64", "type": DataType.INT64},
+            {"name": "float", "type": DataType.FLOAT},
+            {"name": default_binary_vec_field_name, "type": DataType.BINARY_VECTOR, "params": {"dim": default_dim}}
+        ],
+        "segment_row_limit": default_segment_row_limit,
+        "auto_id": auto_id
+    }
+    return default_fields
+
+
+def gen_simple_index():
+    index_params = []
+    for i in range(len(all_index_types)):
+        if all_index_types[i] in binary_support:
+            continue
+        dic = {"index_type": all_index_types[i], "metric_type": "L2"}
+        dic.update({"params": default_index_params[i]})
+        index_params.append(dic)
+    return index_params
+
+
+def get_vectors(num, dim, is_normal=True):
+    vectors = [[random.random() for _ in range(dim)] for _ in range(num)]
+    vectors = preprocessing.normalize(vectors, axis=1, norm='l2')
+    return vectors.tolist()
+
+
+def get_entities(nb=default_nb, is_normal=False):
+    vectors = get_vectors(nb, default_dim, is_normal)
+    entities = [
+        {"name": "int64", "type": DataType.INT64, "values": [i for i in range(nb)]},
+        {"name": "float", "type": DataType.FLOAT, "values": [float(i) for i in range(nb)]},
+        {"name": default_float_vec_field_name, "type": DataType.FLOAT_VECTOR, "values": vectors}
+    ]
+    return entities
+
+
+def get_unique_str(str_value="test_"):
+    prefix = "".join(random.choice(string.ascii_letters + string.digits) for _ in range(8))
+    return str_value + "_" + prefix
+
+
+def modify_file(file_name_list, input_content=""):
+    if not isinstance(file_name_list, list):
+        log.error("[modify_file] file is not a list.")
+
+    for file_name in file_name_list:
+        if not os.path.isfile(file_name):
+            log.error("[modify_file] file(%s) is not exist." % file_name)
+
+        with open(file_name, "r+") as f:
+            f.seek(0)
+            f.truncate()
+            f.write(input_content)
+            f.close()
+
+    log.info("[modify_file] File(%s) modification is complete." % file_name_list)

tests20/python_client/common/common_type.py

+
+""" Initialized parameters """
+port = 19530
+epsilon = 0.000001
+namespace = "milvus"
+default_flush_interval = 1
+big_flush_interval = 1000
+default_drop_interval = 3
+default_dim = 128
+default_nb = 3000
+default_top_k = 10
+max_top_k = 16384
+max_partition_num = 4096  # 256
+default_segment_row_limit = 1000
+default_server_segment_row_limit = 1024 * 512
+default_float_vec_field_name = "float_vector"
+default_binary_vec_field_name = "binary_vector"
+default_partition_name = "_default"
+default_tag = "1970_01_01"
+row_count = "row_count"
+
+
+"""" List of parameters used to pass """
+get_invalid_strs = [[], 1, [1, "2", 3], (1,), {1: 1}, None, "12-s", "12 s", "(mn)", "中文", "%$#",
+                    "a".join("a" for i in range(256))]
+
+
+""" Specially defined list """
+all_index_types = ["FLAT", "IVF_FLAT", "IVF_SQ8", "IVF_PQ", "HNSW", "ANNOY", "RHNSW_FLAT", "RHNSW_PQ", "RHNSW_SQ",
+                   "BIN_FLAT", "BIN_IVF_FLAT"]
+
+default_index_params = [{"nlist": 128}, {"nlist": 128}, {"nlist": 128}, {"nlist": 128, "m": 16, "nbits": 8},
+                        {"M": 48, "efConstruction": 500}, {"n_trees": 50}, {"M": 48, "efConstruction": 500},
+                        {"M": 48, "efConstruction": 500, "PQM": 64}, {"M": 48, "efConstruction": 500}, {"nlist": 128},
+                        {"nlist": 128}]
+
+Handler_type = ["GRPC", "HTTP"]
+index_cpu_not_support = ["IVF_SQ8_HYBRID"]
+binary_support = ["BIN_FLAT", "BIN_IVF_FLAT"]
+delete_support = ["FLAT", "IVF_FLAT", "IVF_SQ8", "IVF_SQ8_HYBRID", "IVF_PQ"]
+ivf = ["FLAT", "IVF_FLAT", "IVF_SQ8", "IVF_SQ8_HYBRID", "IVF_PQ"]
+skip_pq = ["IVF_PQ", "RHNSW_PQ", "RHNSW_SQ"]
+binary_metrics = ["JACCARD", "HAMMING", "TANIMOTO", "SUBSTRUCTURE", "SUPERSTRUCTURE"]
+structure_metrics = ["SUBSTRUCTURE", "SUPERSTRUCTURE"]
+
+
+""" The name of the method used to check the result """
+cname_param_check = "cname_param_check"
+pname_param_check = "pname_param_check"
+
+
+class CaseLabel:
+    """
+    Testcase Levels
+    CI Regression:
+        L0:
+            part of CI Regression
+            triggered by github commit
+            optional used for dev to verify his fix before submitting a PR(like smoke)
+            ~100 testcases and run in 3 mins
+        L1:
+            part of CI Regression
+            triggered by github commit
+            must pass before merge
+            run in 15 mins
+    Benchmark:
+        L2:
+            E2E tests and bug-fix verification
+            Nightly run triggered by cron job
+            run in 60 mins
+        L3:
+            Stability/Performance/reliability, etc. special tests
+            Triggered by cron job or manually
+            run duration depends on test configuration
+    """
+    L0 = "L0"
+    L1 = "L1"
+    L2 = "L2"
+    L3 = "L3"

tests20/python_client/config/my_info.py

+import json
+
+
+class MyInfo:
+    def __init__(self):
+        self.get_default_config()
+
+    def get_default_config(self):
+        """ Make sure the path exists """
+        self.home_dir = "/tmp/"
+        self.log_dir = self.home_dir + "log/"
+        self.test_log = "%s/refactor_test.log" % self.log_dir
+        self.test_err = "%s/refactor_test.err" % self.log_dir
+
+
+my_info = MyInfo()

tests20/python_client/conftest.py

+import pytest
+
+
+def pytest_addoption(parser):
+    parser.addoption("--ip", action="store", default="localhost", help="service's ip")
+    parser.addoption("--service", action="store", default="", help="service address")
+    parser.addoption("--port", action="store", default=19530, help="service's port")
+    parser.addoption("--http_port", action="store", default=19121, help="http's port")
+    parser.addoption("--handler", action="store", default="GRPC", help="handler of request")
+    parser.addoption("--tag", action="store", default="all", help="only run tests matching the tag.")
+    parser.addoption('--dry_run', action='store_true', default=False, help="")
+
+
+@pytest.fixture
+def ip(request):
+    return request.config.getoption("--ip")
+
+
+@pytest.fixture
+def service(request):
+    return request.config.getoption("--service")
+
+
+@pytest.fixture
+def port(request):
+    return request.config.getoption("--port")
+
+
+@pytest.fixture
+def http_port(request):
+    return request.config.getoption("--http_port")
+
+
+@pytest.fixture
+def handler(request):
+    return request.config.getoption("--handler")
+
+
+@pytest.fixture
+def tag(request):
+    return request.config.getoption("--tag")
+
+
+@pytest.fixture
+def dry_run(request):
+    return request.config.getoption("--dry_run")
+
+
+

tests20/python_client/pytest.ini

+[pytest]
+
+addopts = --ip 192.168.1.240 --html=/Users/wt/Desktop/report.html
+#  python3 -W ignore -m pytest
\ No newline at end of file

tests20/python_client/requirements.txt

+pytest==5.3.4
+sklearn==0.0
+numpy==1.18.1
+pymilvus-orm==0.0.1
+git+https://github.com/Projectplace/pytest-tags

tests20/python_client/testcases/test_collection.py

+import pytest
+from base.ClientRequest import ApiReq
+from utils.util_log import my_log as log
+from common.common_type import *
+
+
+class TestCollection(ApiReq):
+    """ Test case of collection interface """
+
+    @pytest.mark.tags(CaseLabel.L3)
+    def test_case(self):
+        log.info("Test case of collection interface")

tests20/python_client/testcases/test_connection.py

+import pytest
+from base.ClientRequest import ApiReq
+from utils.util_log import my_log as log
+from common.common_type import *
+
+
+class TestConnection(ApiReq):
+    """ Test case of connections interface """
+
+    @pytest.mark.tags(CaseLabel.L3)
+    def test_case(self):
+        self.connection.configure(check_res='', default={"host": "192.168.1.240", "port": "19530"})
+        res_ = self.connection.get_connection(alias='default')
+        log.info("res : %s" % str(res_))
+        log.info("self.connection : %s" % str(self.connection))

tests20/python_client/testcases/test_index.py

+import pytest
+from base.ClientRequest import ApiReq
+from utils.util_log import my_log as log
+from common.common_type import *
+
+
+class TestIndex(ApiReq):
+    """ Test case of index interface """
+
+    @pytest.mark.tags(CaseLabel.L3)
+    def test_case(self):
+        log.info("Test case of index interface")

tests20/python_client/testcases/test_params.py

+import pytest
+from milvus import DataType
+
+from common.common_type import *
+from common.common_func import *
+from base.ClientRequest import ApiReq
+from utils.util_log import my_log as log
+
+
+class TestParams(ApiReq):
+    def test_1(self):
+        self.connection.configure(check_res='', default={"host": "192.168.1.240", "port": "19530"})
+        res_ = self.connection.get_connection(alias='default')
+        log.info("res : %s" % str(res_))
+        log.info("self.connection : %s" % str(self.connection))
+        log.info("self.collection : %s" % str(self.collection))
+        log.info("self.partition : %s" % str(self.partition))
+        log.info("self.index : %s" % str(self.index))
+        log.info("self.utility : %s" % str(self.utility))
+
+
+#     @pytest.mark.parametrize("collection_name", get_invalid_strs)
+#     @pytest.mark.parametrize("fields", [get_binary_default_fields()])
+#     @pytest.mark.parametrize("partition_tag, field_name, params, entities",
+#                              [(default_tag, default_float_vec_field_name, gen_simple_index()[0], get_entities()[0])])
+#     def test_collection_name_params_check(self, collection_name, fields, partition_tag, field_name, params, entities):
+#
+#         self.create_collection(collection_name, fields, check_res=cname_param_check)
+#         self.get_collection_stats(collection_name, check_res=cname_param_check)
+#         self.flush(collection_name, check_res=cname_param_check)
+#
+#         self.drop_collection(collection_name, check_res=cname_param_check)
+#         self.has_collection(collection_name, check_res=cname_param_check)
+#         self.describe_collection(collection_name, check_res=cname_param_check)
+#         self.load_collection(collection_name, check_res=cname_param_check)
+#         self.release_collection(collection_name, check_res=cname_param_check)
+#
+#         self.create_partition(collection_name, partition_tag, check_res=cname_param_check)
+#         self.drop_partition(collection_name, partition_tag, check_res=cname_param_check)
+#         self.has_partition(collection_name, partition_tag, check_res=cname_param_check)
+#         self.load_partitions(collection_name, partition_tag, check_res=cname_param_check)
+#         self.release_partitions(collection_name, partition_tag, check_res=cname_param_check)
+#         self.list_partitions(collection_name, check_res=cname_param_check)
+#
+#         self.create_index(collection_name, field_name, params, check_res=cname_param_check)
+#         self.drop_index(collection_name, field_name, check_res=cname_param_check)
+#         self.describe_index(collection_name, field_name, check_res=cname_param_check)
+#         self.search(collection_name, dsl=[1], check_res=cname_param_check)
+#
+#         # self.insert(collection_name, entities=[1])
+#
+#     @pytest.mark.parametrize("collection_name, fields", [(get_unique_str(), get_binary_default_fields())])
+#     @pytest.mark.parametrize("partition_tag", get_invalid_strs)
+#     def test_partition_tag_params_check(self, collection_name, fields, partition_tag):
+#         self.create_collection(collection_name, fields)
+#
+#         self.create_partition(collection_name, partition_tag, check_res=pname_param_check)
+#         self.drop_partition(collection_name, partition_tag, check_res=pname_param_check)
+#         self.has_partition(collection_name, partition_tag, check_res=pname_param_check)
+#         """No parameter check"""
+#         # self.load_partitions(collection_name, partition_tag, check_res=None)
+#         # self.release_partitions(collection_name, partition_tag, check_res=None)
+#
+#         self.drop_collection(collection_name)

tests20/python_client/testcases/test_partition.py

+import pytest
+from base.ClientRequest import ApiReq
+from utils.util_log import my_log as log
+from common.common_type import *
+
+
+class TestPartition(ApiReq):
+    """ Test case of partition interface """
+
+    @pytest.mark.tags(CaseLabel.L3)
+    def test_case(self):
+        log.info("Test case of partition interface")

tests20/python_client/testcases/test_schema.py

+import pytest
+from base.ClientRequest import ApiReq
+from utils.util_log import my_log as log
+from common.common_type import *
+
+
+class TestSchema(ApiReq):
+    """ Test case of schema interface """
+
+    @pytest.mark.tags(CaseLabel.L3)
+    def test_case(self):
+        log.info("Test case of schema interface")

tests20/python_client/testcases/test_search.py

+import pytest
+from base.ClientRequest import ApiReq
+from utils.util_log import my_log as log
+from common.common_type import *
+
+
+class TestSearch(ApiReq):
+    """ Test case of search interface """
+
+    @pytest.mark.tags(CaseLabel.L3)
+    def test_case(self):
+        log.info("Test case of search interface")

tests20/python_client/testcases/test_template.py

+import pytest
+from milvus import DataType
+
+
+from common.common_type import *
+from common.common_func import *
+from base.ClientRequest import Base, ApiReq
+
+
+class CheckParams:
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def check_str_param(self):
+        pass
\ No newline at end of file

tests20/python_client/testcases/test_utility.py

+import pytest
+from base.ClientRequest import ApiReq
+from utils.util_log import my_log as log
+from common.common_type import *
+
+
+class TestUtility(ApiReq):
+    """ Test case of utility interface """
+
+    @pytest.mark.tags(CaseLabel.L3)
+    def test_case(self):
+        log.info("Test case of utility interface")

tests20/python_client/utils/util_log.py

+import logging
+
+from config.my_info import my_info
+
+
+class MyLog:
+    def __init__(self, logger, log_file, log_err):
+        self.logger = logger
+        self.log_file = log_file
+        self.log_err = log_err
+
+        self.log = logging.getLogger(self.logger)
+        self.log.setLevel(logging.DEBUG)
+
+        try:
+            fh = logging.FileHandler(log_file)
+            fh.setLevel(logging.DEBUG)
+            formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+            fh.setFormatter(formatter)
+            self.log.addHandler(fh)
+
+            eh = logging.FileHandler(log_err)
+            eh.setLevel(logging.ERROR)
+            formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+            eh.setFormatter(formatter)
+            self.log.addHandler(eh)
+
+        except Exception as e:
+            print("Can not use %s or %s to log." % (log_file, log_err))
+
+
+"""All modules share this unified log"""
+test_log = my_info.test_log
+test_err = my_info.test_err
+my_log = MyLog('refactor_test', test_log, test_err).log