Update group process

Former-commit-id: 6c4cae438a66556cfbb0c34d7ec3fce85d89c844

pyengine/engine/controller/meta_manager.py

@@ -15,7 +15,6 @@ class MetaManager(object):
 
         # add into database
         db.session.add(new_group)
-        db.session.commit()
 
         return ErrorCode.SUCCESS_CODE, group_name
 
@@ -27,6 +26,11 @@ class MetaManager(object):
         else:
             return ErrorCode.FAULT_CODE, None
 
+    @staticmethod
+    def GetAllGroup():
+        groups = GroupTable.query.all()
+        return groups
+
     @staticmethod
     def DeleteGroup(group):
         db.session.delete(group)
@@ -35,6 +39,6 @@ class MetaManager(object):
     def DeleteGroupFiles(group_name):
         records = FileTable.query.filter(FileTable.group_name == group_name).all()
         for record in records:
-            print("record.group_name: ", record.group_name)
+            # print("record.group_name: ", record.group_name)
             db.session.delete(record)
     
\ No newline at end of file

pyengine/engine/controller/tests/test_vector_engine.py

 from engine.controller.vector_engine import VectorEngine
 from engine.settings import DATABASE_DIRECTORY
+from engine.controller.error_code import ErrorCode
 from flask import jsonify
 import pytest
 import os
@@ -22,23 +23,22 @@ class TestVectorEngine:
     def test_group(self):
         # Make sure there is no group
         code, group_id = VectorEngine.DeleteGroup('test_group')
-        assert code == VectorEngine.SUCCESS_CODE
+        assert code == ErrorCode.SUCCESS_CODE
         assert group_id == 'test_group'
 
         # Add a group
         code, group_id = VectorEngine.AddGroup('test_group', 8)
-        assert code == VectorEngine.SUCCESS_CODE
+        assert code == ErrorCode.SUCCESS_CODE
         assert group_id == 'test_group'
 
         # Check the group existing
         code, group_id = VectorEngine.GetGroup('test_group')
-        assert code == VectorEngine.SUCCESS_CODE
+        assert code == ErrorCode.SUCCESS_CODE
         assert group_id == 'test_group'
 
         # Check the group list
         code, group_list = VectorEngine.GetGroupList()
-        assert code == VectorEngine.SUCCESS_CODE
-        print("group_list: ", group_list)
+        assert code == ErrorCode.SUCCESS_CODE
         assert group_list == [{'group_name': 'test_group', 'file_number': 0}]
 
         # Add Vector for not exist group
@@ -48,21 +48,21 @@ class TestVectorEngine:
 
         # Add vector for exist group
         code, vector_id = VectorEngine.AddVector('test_group', self.__vectors)
-        assert code == VectorEngine.SUCCESS_CODE
+        assert code == ErrorCode.SUCCESS_CODE
         assert vector_id == ['test_group.0', 'test_group.1', 'test_group.2', 'test_group.3', 'test_group.4', 'test_group.5', 'test_group.6', 'test_group.7', 'test_group.8', 'test_group.9']
 
         # Check search vector interface
         code, vector_id = VectorEngine.SearchVector('test_group', self.__vector, self.__limit)
-        assert code == VectorEngine.SUCCESS_CODE
+        assert code == ErrorCode.SUCCESS_CODE
         assert vector_id == ['test_group.0']
 
         # Check create index interface
         code = VectorEngine.CreateIndex('test_group')
-        assert code == VectorEngine.SUCCESS_CODE
+        assert code == ErrorCode.SUCCESS_CODE
 
         # Remove the group
         code, group_id = VectorEngine.DeleteGroup('test_group')
-        assert code == VectorEngine.SUCCESS_CODE
+        assert code == ErrorCode.SUCCESS_CODE
         assert group_id == 'test_group'
 
         # Check the group is disppeared
@@ -81,7 +81,7 @@ class TestVectorEngine:
 
         # Clear raw file
         code = VectorEngine.ClearRawFile('test_group')
-        assert code == VectorEngine.SUCCESS_CODE
+        assert code == ErrorCode.SUCCESS_CODE
 
     def test_raw_file(self):
         filename = VectorEngine.InsertVectorIntoRawFile('test_group', 'test_group.raw', self.__vector, 0)
@@ -99,7 +99,7 @@ class TestVectorEngine:
         assert np.all(vector_list == expected_list)
 
         code = VectorEngine.ClearRawFile('test_group')
-        assert code == VectorEngine.SUCCESS_CODE
+        assert code == ErrorCode.SUCCESS_CODE
 
 
 

pyengine/engine/controller/vector_engine.py

@@ -29,6 +29,7 @@ class VectorEngine(object):
         else:
             StorageManager.AddGroup(group_name)
             MetaManager.AddGroup(group_name, dimension)
+            MetaManager.Sync()
             return ErrorCode.SUCCESS_CODE, group_name
 
 
@@ -43,13 +44,8 @@ class VectorEngine(object):
         if(group):
             MetaManager.DeleteGroup(group)
             StorageManager.DeleteGroup(group_name)
-
-            records = FileTable.query.filter(FileTable.group_name == group_name).all()
-            for record in records:
-                print("record.group_name: ", record.group_name)
-                db.session.delete(record)
-            db.session.commit()
-
+            MetaManager.DeleteGroupFiles(group_name)
+            MetaManager.Sync()
             return VectorEngine.SUCCESS_CODE, group_name
         else:
             return VectorEngine.SUCCESS_CODE, group_name
@@ -57,40 +53,39 @@ class VectorEngine(object):
 
     @staticmethod
     def GetGroupList():
-        group = GroupTable.query.all()
+        groups = MetaManager.GetAllGroup()
         group_list = []
-        for group_tuple in group:
+        for group_tuple in groups:
             group_item = {}
             group_item['group_name'] = group_tuple.group_name
-            group_item['file_number'] = group_tuple.file_number
+            group_item['file_number'] = 0
             group_list.append(group_item)
 
-        print(group_list)
         return VectorEngine.SUCCESS_CODE, group_list
 
 
     @staticmethod
-    def AddVector(group_id, vectors):
-        print(group_id, vectors)
-        code, _, = VectorEngine.GetGroup(group_id)
-        if code == VectorEngine.FAULT_CODE:
+    def AddVector(group_name, vectors):
+        print(group_name, vectors)
+        error, _ = MetaManager.GetGroup(group_name)
+        if error == VectorEngine.FAULT_CODE:
             return VectorEngine.GROUP_NOT_EXIST, 'invalid'
 
         vector_str_list = []
         for vector in vectors:
-            file = FileTable.query.filter(FileTable.group_name == group_id).filter(FileTable.type == 'raw').first()
-            group = GroupTable.query.filter(GroupTable.group_name == group_id).first()
+            file = FileTable.query.filter(FileTable.group_name == group_name).filter(FileTable.type == 'raw').first()
+            group = GroupTable.query.filter(GroupTable.group_name == group_name).first()
 
             if file:
                 print('insert into exist file')
                 # create vector id
                 vector_id = file.seq_no + 1
                 # insert into raw file
-                VectorEngine.InsertVectorIntoRawFile(group_id, file.filename, vector, vector_id)
+                VectorEngine.InsertVectorIntoRawFile(group_name, file.filename, vector, vector_id)
 
                 # check if the file can be indexed
                 if file.row_number + 1 >= ROW_LIMIT:
-                    raw_vector_array, raw_vector_id_array = VectorEngine.GetVectorListFromRawFile(group_id)
+                    raw_vector_array, raw_vector_id_array = VectorEngine.GetVectorListFromRawFile(group_name)
                     d = group.dimension
 
                     # create index
@@ -101,7 +96,7 @@ class VectorEngine(object):
                     index_filename = file.filename + '_index'
                     serialize.write_index(file_name=index_filename, index=index)
 
-                    FileTable.query.filter(FileTable.group_name == group_id).filter(FileTable.type == 'raw').update({'row_number':file.row_number + 1,
+                    FileTable.query.filter(FileTable.group_name == group_name).filter(FileTable.type == 'raw').update({'row_number':file.row_number + 1,
                                                                                                                     'type': 'index',
                                                                                                                     'filename': index_filename,
                                                                                                                     'seq_no': file.seq_no + 1})
@@ -109,7 +104,7 @@ class VectorEngine(object):
                     VectorEngine.group_dict = None
                 else:
                     # we still can insert into exist raw file, update database
-                    FileTable.query.filter(FileTable.group_name == group_id).filter(FileTable.type == 'raw').update({'row_number':file.row_number + 1, 
+                    FileTable.query.filter(FileTable.group_name == group_name).filter(FileTable.type == 'raw').update({'row_number':file.row_number + 1, 
                                                                                                                     'seq_no': file.seq_no + 1})
                     db.session.commit()
                     print('Update db for raw file insertion')
@@ -117,16 +112,16 @@ class VectorEngine(object):
             else:
                 print('add a new raw file')
                 # first raw file
-                raw_filename = group_id + '.raw'
+                raw_filename = group_name + '.raw'
                 # create vector id
                 vector_id = 0
                 # create and insert vector into raw file
-                VectorEngine.InsertVectorIntoRawFile(group_id, raw_filename, vector, vector_id)
+                VectorEngine.InsertVectorIntoRawFile(group_name, raw_filename, vector, vector_id)
                 # insert a record into database
-                db.session.add(FileTable(group_id, raw_filename, 'raw', 1))
+                db.session.add(FileTable(group_name, raw_filename, 'raw', 1))
                 db.session.commit()
 
-            vector_str_list.append(group_id + '.' + str(vector_id))
+            vector_str_list.append(group_name + '.' + str(vector_id))
 
         return VectorEngine.SUCCESS_CODE, vector_str_list