提交 9ee91b47 编写于 作者: X Xiaohai Xu

change table to collection

Signed-off-by: NXiaohai Xu <xiaohaix@student.unimelb.edu.au>
上级 f3e7e647
......@@ -106,7 +106,7 @@ def table(request, connect):
def teardown():
status, table_names = connect.show_tables()
for table_name in table_names:
connect.delete_table(table_name)
connect.drop_collection(table_name)
request.addfinalizer(teardown)
......@@ -130,7 +130,7 @@ def ip_table(request, connect):
def teardown():
status, table_names = connect.show_tables()
for table_name in table_names:
connect.delete_table(table_name)
connect.drop_collection(table_name)
request.addfinalizer(teardown)
......@@ -154,7 +154,7 @@ def jac_table(request, connect):
def teardown():
status, table_names = connect.show_tables()
for table_name in table_names:
connect.delete_table(table_name)
connect.drop_collection(table_name)
request.addfinalizer(teardown)
......@@ -177,7 +177,7 @@ def ham_table(request, connect):
def teardown():
status, table_names = connect.show_tables()
for table_name in table_names:
connect.delete_table(table_name)
connect.drop_collection(table_name)
request.addfinalizer(teardown)
......@@ -200,7 +200,7 @@ def tanimoto_table(request, connect):
def teardown():
status, table_names = connect.show_tables()
for table_name in table_names:
connect.delete_table(table_name)
connect.drop_collection(table_name)
request.addfinalizer(teardown)
......
......@@ -18,7 +18,7 @@ nb = 6000
nlist = 1024
class TestTableInfoBase:
class TestCollectionInfoBase:
def index_string_convert(self, index_string, index_type):
if index_string == "IDMAP" and index_type == IndexType.FLAT:
return True
......@@ -30,87 +30,87 @@ class TestTableInfoBase:
"""
******************************************************************
The following cases are used to test `table_info` function
The following cases are used to test `collection_info` function
******************************************************************
"""
@pytest.mark.timeout(INFO_TIMEOUT)
def test_get_table_info_name_None(self, connect, table):
def test_get_collection_info_name_None(self, connect, collection):
'''
target: get table info where table name is None
method: call table_info with the table_name: None
target: get collection info where collection name is None
method: call collection_info with the collection_name: None
expected: status not ok
'''
table_name = None
status, info = connect.table_info(table_name)
collection_name = None
status, info = connect.collection_info(collection_name)
assert not status.OK()
@pytest.mark.timeout(INFO_TIMEOUT)
def test_get_table_info_name_not_existed(self, connect, table):
def test_get_collection_info_name_not_existed(self, connect, collection):
'''
target: get table info where table name does not exist
method: call table_info with a random table_name, which is not in db
target: get collection info where collection name does not exist
method: call collection_info with a random collection_name, which is not in db
expected: status not ok
'''
table_name = gen_unique_str("not_existed_table")
status, info = connect.table_info(table_name)
collection_name = gen_unique_str("not_existed_collection")
status, info = connect.collection_info(collection_name)
assert not status.OK()
@pytest.fixture(
scope="function",
params=gen_invalid_table_names()
params=gen_invalid_collection_names()
)
def get_table_name(self, request):
def get_collection_name(self, request):
yield request.param
@pytest.mark.timeout(INFO_TIMEOUT)
def test_get_table_info_name_invalid(self, connect, get_table_name):
def test_get_collection_info_name_invalid(self, connect, get_collection_name):
'''
target: get table info where table name is invalid
method: call table_info with invalid table_name
target: get collection info where collection name is invalid
method: call collection_info with invalid collection_name
expected: status not ok
'''
table_name = get_table_name
status, info = connect.table_info(table_name)
collection_name = get_collection_name
status, info = connect.collection_info(collection_name)
assert not status.OK()
@pytest.mark.timeout(INFO_TIMEOUT)
def test_get_table_info_table_row_count(self, connect, table):
def test_get_collection_info_collection_row_count(self, connect, collection):
'''
target: get row count with table_info
method: add and delete vectors, check count in table info
target: get row count with collection_info
method: add and delete vectors, check count in collection info
expected: status ok, count as expected
'''
vectors = gen_vector(nb, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, info = connect.table_info(table)
status, info = connect.collection_info(collection)
assert status.OK()
assert info.count == nb
# delete a few vectors
delete_ids = [ids[0], ids[-1]]
status = connect.delete_by_id(table, delete_ids)
status = connect.delete_by_id(collection, delete_ids)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, info = connect.table_info(table)
status, info = connect.collection_info(collection)
assert status.OK()
assert info.count == nb - 2
@pytest.mark.timeout(INFO_TIMEOUT)
def test_get_table_info_partition_stats_A(self, connect, table):
def test_get_collection_info_partition_stats_A(self, connect, collection):
'''
target: get partition info in a table
method: no partition, call table_info and check partition_stats
target: get partition info in a collection
method: no partition, call collection_info and check partition_stats
expected: status ok, "_default" partition is listed
'''
vectors = gen_vector(nb, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, info = connect.table_info(table)
status, info = connect.collection_info(collection)
assert status.OK()
logging.getLogger().info(info)
assert len(info.partitions_stat) == 1
......@@ -119,19 +119,19 @@ class TestTableInfoBase:
@pytest.mark.timeout(INFO_TIMEOUT)
def test_get_table_info_partition_stats_B(self, connect, table):
def test_get_collection_info_partition_stats_B(self, connect, collection):
'''
target: get partition info in a table
method: call table_info after partition created and check partition_stats
target: get partition info in a collection
method: call collection_info after partition created and check partition_stats
expected: status ok, vectors added to partition
'''
vectors = gen_vectors(nb, dim)
status = connect.create_partition(table, tag)
status, ids = connect.add_vectors(table, vectors, partition_tag=tag)
status = connect.create_partition(collection, tag)
status, ids = connect.add_vectors(collection, vectors, partition_tag=tag)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, info = connect.table_info(table)
status, info = connect.collection_info(collection)
assert status.OK()
logging.getLogger().info(info)
assert len(info.partitions_stat) == 2
......@@ -139,23 +139,23 @@ class TestTableInfoBase:
assert info.partitions_stat[1].count == nb
@pytest.mark.timeout(INFO_TIMEOUT)
def test_get_table_info_partition_stats_C(self, connect, table):
def test_get_collection_info_partition_stats_C(self, connect, collection):
'''
target: get partition info in a table
method: create two partitions, add vectors in one of the partitions, call table_info and check
target: get partition info in a collection
method: create two partitions, add vectors in one of the partitions, call collection_info and check
expected: status ok, vectors added to one partition but not the other
'''
new_tag = "new_tag"
vectors = gen_vectors(nb, dim)
status = connect.create_partition(table, tag)
status = connect.create_partition(collection, tag)
assert status.OK()
status = connect.create_partition(table, new_tag)
status = connect.create_partition(collection, new_tag)
assert status.OK()
status, ids = connect.add_vectors(table, vectors, partition_tag=tag)
status, ids = connect.add_vectors(collection, vectors, partition_tag=tag)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, info = connect.table_info(table)
status, info = connect.collection_info(collection)
assert status.OK()
logging.getLogger().info(info)
for partition in info.partitions_stat:
......@@ -165,25 +165,25 @@ class TestTableInfoBase:
assert partition.count == 0
@pytest.mark.timeout(INFO_TIMEOUT)
def test_get_table_info_partition_stats_D(self, connect, table):
def test_get_collection_info_partition_stats_D(self, connect, collection):
'''
target: get partition info in a table
method: create two partitions, add vectors in both partitions, call table_info and check
target: get partition info in a collection
method: create two partitions, add vectors in both partitions, call collection_info and check
expected: status ok, vectors added to both partitions
'''
new_tag = "new_tag"
vectors = gen_vectors(nb, dim)
status = connect.create_partition(table, tag)
status = connect.create_partition(collection, tag)
assert status.OK()
status = connect.create_partition(table, new_tag)
status = connect.create_partition(collection, new_tag)
assert status.OK()
status, ids = connect.add_vectors(table, vectors, partition_tag=tag)
status, ids = connect.add_vectors(collection, vectors, partition_tag=tag)
assert status.OK()
status, ids = connect.add_vectors(table, vectors, partition_tag=new_tag)
status, ids = connect.add_vectors(collection, vectors, partition_tag=new_tag)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, info = connect.table_info(table)
status, info = connect.collection_info(collection)
assert status.OK()
assert info.count == nb * 2
for partition in info.partitions_stat:
......@@ -205,24 +205,24 @@ class TestTableInfoBase:
return request.param
@pytest.mark.timeout(INFO_TIMEOUT)
def test_get_table_info_after_index_created(self, connect, table, get_simple_index):
def test_get_collection_info_after_index_created(self, connect, collection, get_simple_index):
'''
target: test table info after index created
method: create table, add vectors, create index and call table_info
target: test collection info after index created
method: create collection, add vectors, create index and call collection_info
expected: status ok, index created and shown in segments_stat
'''
index_param = get_simple_index["index_param"]
index_type = get_simple_index["index_type"]
vectors = gen_vector(nb, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status = connect.create_index(table, index_type, index_param)
status = connect.create_index(collection, index_type, index_param)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, info = connect.table_info(table)
status, info = connect.collection_info(collection)
assert status.OK()
logging.getLogger().info(info)
index_string = info.partitions_stat[0].segments_stat[0].index_name
......@@ -231,24 +231,24 @@ class TestTableInfoBase:
assert nb == info.partitions_stat[0].segments_stat[0].count
@pytest.mark.timeout(INFO_TIMEOUT)
def test_get_table_info_after_create_same_index_repeatedly(self, connect, table, get_simple_index):
def test_get_collection_info_after_create_same_index_repeatedly(self, connect, collection, get_simple_index):
'''
target: test table info after index created repeatedly
method: create table, add vectors, create index and call table_info multiple times
target: test collection info after index created repeatedly
method: create collection, add vectors, create index and call collection_info multiple times
expected: status ok, index info shown in segments_stat
'''
index_param = get_simple_index["index_param"]
index_type = get_simple_index["index_type"]
vectors = gen_vector(nb, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status = connect.create_index(table, index_type, index_param)
status = connect.create_index(table, index_type, index_param)
status = connect.create_index(table, index_type, index_param)
status = connect.create_index(collection, index_type, index_param)
status = connect.create_index(collection, index_type, index_param)
status = connect.create_index(collection, index_type, index_param)
assert status.OK()
status, info = connect.table_info(table)
status, info = connect.collection_info(collection)
assert status.OK()
logging.getLogger().info(info)
index_string = info.partitions_stat[0].segments_stat[0].index_name
......@@ -257,22 +257,22 @@ class TestTableInfoBase:
assert nb == info.partitions_stat[0].segments_stat[0].count
@pytest.mark.timeout(INFO_TIMEOUT)
def test_get_table_info_after_create_different_index_repeatedly(self, connect, table, get_simple_index):
def test_get_collection_info_after_create_different_index_repeatedly(self, connect, collection, get_simple_index):
'''
target: test table info after index created repeatedly
method: create table, add vectors, create index and call table_info multiple times
target: test collection info after index created repeatedly
method: create collection, add vectors, create index and call collection_info multiple times
expected: status ok, index info shown in segments_stat
'''
vectors = gen_vector(nb, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
index_param = {"nlist": nlist}
for index_type in [IndexType.FLAT, IndexType.IVFLAT, IndexType.IVF_SQ8]:
status = connect.create_index(table, index_type, index_param)
status = connect.create_index(collection, index_type, index_param)
assert status.OK()
status, info = connect.table_info(table)
status, info = connect.collection_info(collection)
assert status.OK()
logging.getLogger().info(info)
index_string = info.partitions_stat[0].segments_stat[0].index_name
......
......@@ -285,7 +285,7 @@ class TestConnect:
b. data_set not too large incase disconnection happens when data is underd-preparing
c. data_set not too small incase disconnection happens when data has already been transferred
d. make sure disconnection happens when data is in-transport
Expected: Failure, get_table_row_count == 0
Expected: Failure, count_collection == 0
'''
pass
......
......@@ -10,7 +10,7 @@ from utils import *
dim = 128
index_file_size = 10
table_id = "test_flush"
collection_id = "test_flush"
DELETE_TIMEOUT = 60
nprobe = 1
tag = "1970-01-01"
......@@ -38,117 +38,117 @@ class TestFlushBase:
pytest.skip("Only support CPU mode")
return request.param
def test_flush_table_not_existed(self, connect, table):
def test_flush_collection_not_existed(self, connect, collection):
'''
target: test delete vector, params table_name not existed
target: test delete vector, params collection_name not existed
method: add vector and delete
expected: status not ok
'''
table_new = gen_unique_str()
status = connect.flush([table_new])
collection_new = gen_unique_str()
status = connect.flush([collection_new])
assert not status.OK()
def test_flush_empty_table(self, connect, table):
def test_flush_empty_collection(self, connect, collection):
'''
method: flush table with no vectors
method: flush collection with no vectors
expected: status ok
'''
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
def test_add_partition_flush(self, connect, table):
def test_add_partition_flush(self, connect, collection):
'''
method: add vectors into partition in table, flush serveral times
method: add vectors into partition in collection, flush serveral times
expected: status ok
'''
vectors = gen_vector(nb, dim)
status = connect.create_partition(table, tag)
status = connect.create_partition(collection, tag)
vectors = gen_vectors(nb, dim)
ids = [i for i in range(nb)]
status, ids = connect.insert(table, vectors, ids)
status = connect.flush([table])
result, res = connect.get_table_row_count(table)
status, ids = connect.insert(collection, vectors, ids)
status = connect.flush([collection])
result, res = connect.count_collection(collection)
assert res == nb
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
result, res = connect.get_table_row_count(table)
result, res = connect.count_collection(collection)
assert res == 2 * nb
def test_add_partitions_flush(self, connect, table):
def test_add_partitions_flush(self, connect, collection):
'''
method: add vectors into partitions in table, flush one
method: add vectors into partitions in collection, flush one
expected: status ok
'''
vectors = gen_vectors(nb, dim)
tag_new = gen_unique_str()
status = connect.create_partition(table, tag)
status = connect.create_partition(table, tag_new)
status = connect.create_partition(collection, tag)
status = connect.create_partition(collection, tag_new)
ids = [i for i in range(nb)]
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
status = connect.flush([table])
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
status = connect.flush([collection])
assert status.OK()
status, ids = connect.insert(table, vectors, ids, partition_tag=tag_new)
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag_new)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
result, res = connect.get_table_row_count(table)
result, res = connect.count_collection(collection)
assert res == 2 * nb
def test_add_tables_flush(self, connect, table):
def test_add_collections_flush(self, connect, collection):
'''
method: add vectors into tables, flush one
method: add vectors into collections, flush one
expected: status ok
'''
vectors = gen_vectors(nb, dim)
table_new = gen_unique_str()
param = {'table_name': table_new,
collection_new = gen_unique_str()
param = {'collection_name': collection_new,
'dimension': dim,
'index_file_size': index_file_size,
'metric_type': MetricType.L2}
status = connect.create_table(param)
status = connect.create_partition(table, tag)
status = connect.create_partition(table_new, tag)
status = connect.create_collection(param)
status = connect.create_partition(collection, tag)
status = connect.create_partition(collection_new, tag)
vectors = gen_vectors(nb, dim)
ids = [i for i in range(nb)]
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
status, ids = connect.insert(table_new, vectors, ids, partition_tag=tag)
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
status, ids = connect.insert(collection_new, vectors, ids, partition_tag=tag)
assert status.OK()
status = connect.flush([table])
status = connect.flush([table_new])
status = connect.flush([collection])
status = connect.flush([collection_new])
assert status.OK()
result, res = connect.get_table_row_count(table)
result, res = connect.count_collection(collection)
assert res == nb
result, res = connect.get_table_row_count(table_new)
result, res = connect.count_collection(collection_new)
assert res == nb
def test_add_flush_multiable_times(self, connect, table):
def test_add_flush_multiable_times(self, connect, collection):
'''
method: add vectors, flush serveral times
expected: status ok
'''
vectors = gen_vectors(nb, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
for i in range(10):
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
query_vecs = [vectors[0], vectors[1], vectors[-1]]
status, res = connect.search_vectors(table, top_k, query_records=query_vecs)
status, res = connect.search_vectors(collection, top_k, query_records=query_vecs)
assert status.OK()
def test_add_flush_auto(self, connect, table):
def test_add_flush_auto(self, connect, collection):
'''
method: add vectors
expected: status ok
'''
vectors = gen_vectors(nb, dim)
ids = [i for i in range(nb)]
status, ids = connect.add_vectors(table, vectors, ids)
status, ids = connect.add_vectors(collection, vectors, ids)
assert status.OK()
time.sleep(2)
status, res = connect.get_table_row_count(table)
status, res = connect.count_collection(collection)
assert status.OK()
assert res == nb
......@@ -163,7 +163,7 @@ class TestFlushBase:
yield request.param
# both autoflush / flush
def test_add_flush_same_ids(self, connect, table, same_ids):
def test_add_flush_same_ids(self, connect, collection, same_ids):
'''
method: add vectors, with same ids, count(same ids) < 15, > 15
expected: status ok
......@@ -173,79 +173,79 @@ class TestFlushBase:
for i, item in enumerate(ids):
if item <= same_ids:
ids[i] = 0
status, ids = connect.add_vectors(table, vectors, ids)
status, ids = connect.add_vectors(collection, vectors, ids)
time.sleep(2)
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, res = connect.get_table_row_count(table)
status, res = connect.count_collection(collection)
assert status.OK()
assert res == nb
def test_delete_flush_multiable_times(self, connect, table):
def test_delete_flush_multiable_times(self, connect, collection):
'''
method: delete vectors, flush serveral times
expected: status ok
'''
vectors = gen_vectors(nb, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
status = connect.delete_by_id(table, [ids[-1]])
status = connect.delete_by_id(collection, [ids[-1]])
assert status.OK()
for i in range(10):
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
query_vecs = [vectors[0], vectors[1], vectors[-1]]
status, res = connect.search_vectors(table, top_k, query_records=query_vecs)
status, res = connect.search_vectors(collection, top_k, query_records=query_vecs)
assert status.OK()
# TODO: CI fail, LOCAL pass
def _test_table_count_during_flush(self, connect, args):
def _test_collection_count_during_flush(self, connect, args):
'''
method: flush table at background, call `get_table_row_count`
method: flush collection at background, call `count_collection`
expected: status ok
'''
table = gen_unique_str()
collection = gen_unique_str()
uri = "tcp://%s:%s" % (args["ip"], args["port"])
param = {'table_name': table,
param = {'collection_name': collection,
'dimension': dim,
'index_file_size': index_file_size,
'metric_type': MetricType.L2}
milvus = get_milvus()
milvus.connect(uri=uri)
milvus.create_table(param)
milvus.create_collection(param)
vectors = gen_vector(nb, dim)
status, ids = milvus.add_vectors(table, vectors, ids=[i for i in range(nb)])
def flush(table_name):
status, ids = milvus.add_vectors(collection, vectors, ids=[i for i in range(nb)])
def flush(collection_name):
milvus = get_milvus()
milvus.connect(uri=uri)
status = milvus.delete_by_id(table_name, [i for i in range(nb)])
status = milvus.delete_by_id(collection_name, [i for i in range(nb)])
assert status.OK()
status = milvus.flush([table_name])
status = milvus.flush([collection_name])
assert status.OK()
p = Process(target=flush, args=(table, ))
p = Process(target=flush, args=(collection, ))
p.start()
status, res = milvus.get_table_row_count(table)
status, res = milvus.count_collection(collection)
assert status.OK()
p.join()
status, res = milvus.get_table_row_count(table)
status, res = milvus.count_collection(collection)
assert status.OK()
logging.getLogger().info(res)
assert res == 0
class TestTableNameInvalid(object):
class TestcollectionNameInvalid(object):
"""
Test adding vectors with invalid table names
Test adding vectors with invalid collection names
"""
@pytest.fixture(
scope="function",
params=gen_invalid_table_names()
params=gen_invalid_collection_names()
)
def get_table_name(self, request):
def get_collection_name(self, request):
yield request.param
@pytest.mark.level(2)
def test_flush_with_invalid_table_name(self, connect, get_table_name):
table_name = get_table_name
def test_flush_with_invalid_collection_name(self, connect, get_collection_name):
collection_name = get_collection_name
with pytest.raises(Exception) as e:
status, result = connect.flush(table_name)
status, result = connect.flush(collection_name)
......@@ -146,8 +146,8 @@ def gen_invalid_uris():
return uris
def gen_invalid_table_names():
table_names = [
def gen_invalid_collection_names():
collection_names = [
"12-s",
"12/s",
" ",
......@@ -166,7 +166,7 @@ def gen_invalid_table_names():
"中文",
"a".join("a" for i in range(256))
]
return table_names
return collection_names
def gen_invalid_top_ks():
......@@ -617,8 +617,8 @@ def get_search_param(index_type):
logging.getLogger().info("Invalid index_type.")
def assert_has_table(conn, table_name):
status, ok = conn.has_table(table_name)
def assert_has_collection(conn, collection_name):
status, ok = conn.has_collection(collection_name)
return status.OK() and ok
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册