diff --git a/pyengine/engine/controller/meta_manager.py b/pyengine/engine/controller/meta_manager.py index 195302a246ce113cf98af9e09d78e76b4d4a1722..e34ca53d842a3bb003e03757a12a21b8f7c23438 100644 --- a/pyengine/engine/controller/meta_manager.py +++ b/pyengine/engine/controller/meta_manager.py @@ -15,7 +15,6 @@ class MetaManager(object): # add into database db.session.add(new_group) - db.session.commit() return ErrorCode.SUCCESS_CODE, group_name @@ -27,6 +26,11 @@ class MetaManager(object): else: return ErrorCode.FAULT_CODE, None + @staticmethod + def GetAllGroup(): + groups = GroupTable.query.all() + return groups + @staticmethod def DeleteGroup(group): db.session.delete(group) @@ -35,6 +39,6 @@ class MetaManager(object): def DeleteGroupFiles(group_name): records = FileTable.query.filter(FileTable.group_name == group_name).all() for record in records: - print("record.group_name: ", record.group_name) + # print("record.group_name: ", record.group_name) db.session.delete(record) \ No newline at end of file diff --git a/pyengine/engine/controller/tests/test_vector_engine.py b/pyengine/engine/controller/tests/test_vector_engine.py index d1774157c644ac3df4937fbb1b573e2ea723276d..aceb3ecfbbb169772326575e93caa2884a199db4 100644 --- a/pyengine/engine/controller/tests/test_vector_engine.py +++ b/pyengine/engine/controller/tests/test_vector_engine.py @@ -1,5 +1,6 @@ from engine.controller.vector_engine import VectorEngine from engine.settings import DATABASE_DIRECTORY +from engine.controller.error_code import ErrorCode from flask import jsonify import pytest import os @@ -22,23 +23,22 @@ class TestVectorEngine: def test_group(self): # Make sure there is no group code, group_id = VectorEngine.DeleteGroup('test_group') - assert code == VectorEngine.SUCCESS_CODE + assert code == ErrorCode.SUCCESS_CODE assert group_id == 'test_group' # Add a group code, group_id = VectorEngine.AddGroup('test_group', 8) - assert code == VectorEngine.SUCCESS_CODE + assert code == ErrorCode.SUCCESS_CODE assert group_id == 'test_group' # Check the group existing code, group_id = VectorEngine.GetGroup('test_group') - assert code == VectorEngine.SUCCESS_CODE + assert code == ErrorCode.SUCCESS_CODE assert group_id == 'test_group' # Check the group list code, group_list = VectorEngine.GetGroupList() - assert code == VectorEngine.SUCCESS_CODE - print("group_list: ", group_list) + assert code == ErrorCode.SUCCESS_CODE assert group_list == [{'group_name': 'test_group', 'file_number': 0}] # Add Vector for not exist group @@ -48,21 +48,21 @@ class TestVectorEngine: # Add vector for exist group code, vector_id = VectorEngine.AddVector('test_group', self.__vectors) - assert code == VectorEngine.SUCCESS_CODE + assert code == ErrorCode.SUCCESS_CODE assert vector_id == ['test_group.0', 'test_group.1', 'test_group.2', 'test_group.3', 'test_group.4', 'test_group.5', 'test_group.6', 'test_group.7', 'test_group.8', 'test_group.9'] # Check search vector interface code, vector_id = VectorEngine.SearchVector('test_group', self.__vector, self.__limit) - assert code == VectorEngine.SUCCESS_CODE + assert code == ErrorCode.SUCCESS_CODE assert vector_id == ['test_group.0'] # Check create index interface code = VectorEngine.CreateIndex('test_group') - assert code == VectorEngine.SUCCESS_CODE + assert code == ErrorCode.SUCCESS_CODE # Remove the group code, group_id = VectorEngine.DeleteGroup('test_group') - assert code == VectorEngine.SUCCESS_CODE + assert code == ErrorCode.SUCCESS_CODE assert group_id == 'test_group' # Check the group is disppeared @@ -81,7 +81,7 @@ class TestVectorEngine: # Clear raw file code = VectorEngine.ClearRawFile('test_group') - assert code == VectorEngine.SUCCESS_CODE + assert code == ErrorCode.SUCCESS_CODE def test_raw_file(self): filename = VectorEngine.InsertVectorIntoRawFile('test_group', 'test_group.raw', self.__vector, 0) @@ -99,7 +99,7 @@ class TestVectorEngine: assert np.all(vector_list == expected_list) code = VectorEngine.ClearRawFile('test_group') - assert code == VectorEngine.SUCCESS_CODE + assert code == ErrorCode.SUCCESS_CODE diff --git a/pyengine/engine/controller/vector_engine.py b/pyengine/engine/controller/vector_engine.py index 8b31be2a71e459388163617df09d70675b14ea91..454c363697328c349f6db61000b540709b3c9f6f 100644 --- a/pyengine/engine/controller/vector_engine.py +++ b/pyengine/engine/controller/vector_engine.py @@ -29,6 +29,7 @@ class VectorEngine(object): else: StorageManager.AddGroup(group_name) MetaManager.AddGroup(group_name, dimension) + MetaManager.Sync() return ErrorCode.SUCCESS_CODE, group_name @@ -43,13 +44,8 @@ class VectorEngine(object): if(group): MetaManager.DeleteGroup(group) StorageManager.DeleteGroup(group_name) - - records = FileTable.query.filter(FileTable.group_name == group_name).all() - for record in records: - print("record.group_name: ", record.group_name) - db.session.delete(record) - db.session.commit() - + MetaManager.DeleteGroupFiles(group_name) + MetaManager.Sync() return VectorEngine.SUCCESS_CODE, group_name else: return VectorEngine.SUCCESS_CODE, group_name @@ -57,40 +53,39 @@ class VectorEngine(object): @staticmethod def GetGroupList(): - group = GroupTable.query.all() + groups = MetaManager.GetAllGroup() group_list = [] - for group_tuple in group: + for group_tuple in groups: group_item = {} group_item['group_name'] = group_tuple.group_name - group_item['file_number'] = group_tuple.file_number + group_item['file_number'] = 0 group_list.append(group_item) - print(group_list) return VectorEngine.SUCCESS_CODE, group_list @staticmethod - def AddVector(group_id, vectors): - print(group_id, vectors) - code, _, = VectorEngine.GetGroup(group_id) - if code == VectorEngine.FAULT_CODE: + def AddVector(group_name, vectors): + print(group_name, vectors) + error, _ = MetaManager.GetGroup(group_name) + if error == VectorEngine.FAULT_CODE: return VectorEngine.GROUP_NOT_EXIST, 'invalid' vector_str_list = [] for vector in vectors: - file = FileTable.query.filter(FileTable.group_name == group_id).filter(FileTable.type == 'raw').first() - group = GroupTable.query.filter(GroupTable.group_name == group_id).first() + file = FileTable.query.filter(FileTable.group_name == group_name).filter(FileTable.type == 'raw').first() + group = GroupTable.query.filter(GroupTable.group_name == group_name).first() if file: print('insert into exist file') # create vector id vector_id = file.seq_no + 1 # insert into raw file - VectorEngine.InsertVectorIntoRawFile(group_id, file.filename, vector, vector_id) + VectorEngine.InsertVectorIntoRawFile(group_name, file.filename, vector, vector_id) # check if the file can be indexed if file.row_number + 1 >= ROW_LIMIT: - raw_vector_array, raw_vector_id_array = VectorEngine.GetVectorListFromRawFile(group_id) + raw_vector_array, raw_vector_id_array = VectorEngine.GetVectorListFromRawFile(group_name) d = group.dimension # create index @@ -101,7 +96,7 @@ class VectorEngine(object): index_filename = file.filename + '_index' serialize.write_index(file_name=index_filename, index=index) - FileTable.query.filter(FileTable.group_name == group_id).filter(FileTable.type == 'raw').update({'row_number':file.row_number + 1, + FileTable.query.filter(FileTable.group_name == group_name).filter(FileTable.type == 'raw').update({'row_number':file.row_number + 1, 'type': 'index', 'filename': index_filename, 'seq_no': file.seq_no + 1}) @@ -109,7 +104,7 @@ class VectorEngine(object): VectorEngine.group_dict = None else: # we still can insert into exist raw file, update database - FileTable.query.filter(FileTable.group_name == group_id).filter(FileTable.type == 'raw').update({'row_number':file.row_number + 1, + FileTable.query.filter(FileTable.group_name == group_name).filter(FileTable.type == 'raw').update({'row_number':file.row_number + 1, 'seq_no': file.seq_no + 1}) db.session.commit() print('Update db for raw file insertion') @@ -117,16 +112,16 @@ class VectorEngine(object): else: print('add a new raw file') # first raw file - raw_filename = group_id + '.raw' + raw_filename = group_name + '.raw' # create vector id vector_id = 0 # create and insert vector into raw file - VectorEngine.InsertVectorIntoRawFile(group_id, raw_filename, vector, vector_id) + VectorEngine.InsertVectorIntoRawFile(group_name, raw_filename, vector, vector_id) # insert a record into database - db.session.add(FileTable(group_id, raw_filename, 'raw', 1)) + db.session.add(FileTable(group_name, raw_filename, 'raw', 1)) db.session.commit() - vector_str_list.append(group_id + '.' + str(vector_id)) + vector_str_list.append(group_name + '.' + str(vector_id)) return VectorEngine.SUCCESS_CODE, vector_str_list