support multiple vdl process when using gunicorn for x2paddle (#1170)

* support multiple vdl process when using gunicorn for x2paddle

* fix a bug

visualdl/component/inference/model_convert_server.py

@@ -13,10 +13,12 @@
 # limitations under the License.
 # =======================================================================
 import base64
+import glob
+import hashlib
 import json
 import os
+import shutil
 import tempfile
-from collections import deque
 from threading import Lock
 
 from flask import request
@@ -27,15 +29,18 @@ from .xarfile import archive
 from .xarfile import unarchive
 from visualdl.server.api import gen_result
 from visualdl.server.api import result
+from visualdl.utils.dir import X2PADDLE_CACHE_PATH
+
+_max_cache_numbers = 200
 
 
 class ModelConvertApi(object):
     def __init__(self):
         self.supported_formats = {'onnx', 'caffe'}
         self.lock = Lock()
-        self.translated_models = deque(
-            maxlen=5)  # used to store user's translated model for download
-        self.request_id = 0  # used to store user's request
+        self.server_count = 0  # we use this variable to count requests handled,
+        # and check the number of files every 100 requests.
+        # If more than _max_cache_numbers files in cache, we delete the last recent used 50 files.
 
     @result()
     def convert_model(self, format):
@@ -48,8 +53,24 @@ class ModelConvertApi(object):
         result['from'] = format
         result['to'] = 'paddle'
         # call x2paddle to convert models
-        with tempfile.TemporaryDirectory(
-                suffix='x2paddle_translated_models') as tmpdirname:
+        hl = hashlib.md5()
+        hl.update(data)
+        identity = hl.hexdigest()
+        result['request_id'] = identity
+        target_path = os.path.join(X2PADDLE_CACHE_PATH, identity)
+        if os.path.exists(target_path):
+            if os.path.exists(
+                    os.path.join(target_path, 'inference_model',
+                                 'model.pdmodel')):  # if data in cache
+                with open(
+                        os.path.join(target_path, 'inference_model',
+                                     'model.pdmodel'), 'rb') as model_fp:
+                    model_encoded = base64.b64encode(
+                        model_fp.read()).decode('utf-8')
+                result['pdmodel'] = model_encoded
+                return result
+        else:
+            os.makedirs(target_path, exist_ok=True)
         with tempfile.NamedTemporaryFile() as fp:
             fp.write(data)
             fp.flush()
@@ -61,7 +82,7 @@ class ModelConvertApi(object):
                         raise RuntimeError(
                             "[ERROR] onnx is not installed, use \"pip install onnx>=1.6.0\"."
                         )
-                        onnx2paddle(fp.name, tmpdirname)
+                    onnx2paddle(fp.name, target_path)
                 elif format == 'caffe':
                     with tempfile.TemporaryDirectory() as unarchivedir:
                         unarchive(fp.name, unarchivedir)
@@ -80,40 +101,52 @@ class ModelConvertApi(object):
                             raise RuntimeError(
                                 ".prototxt or .caffemodel file is missing in your archive file, \
                 please check files uploaded.")
-                            caffe2paddle(prototxt_path, weight_path,
-                                         tmpdirname, None)
+                        caffe2paddle(prototxt_path, weight_path, target_path,
+                                     None)
             except Exception as e:
                 raise RuntimeError(
                     "[Convertion error] {}.\n Please open an issue at \
                         https://github.com/PaddlePaddle/X2Paddle/issues to report your problem."
                     .format(e))
-                with self.lock:
+        with self.lock:  # we need to enter dirname(target_path) to archive,
+            # in case unneccessary directory added in archive.
             origin_dir = os.getcwd()
-                    os.chdir(os.path.dirname(tmpdirname))
-                    archive_path = os.path.join(
-                        os.path.dirname(tmpdirname),
-                        archive(os.path.basename(tmpdirname)))
+            os.chdir(os.path.dirname(target_path))
+            archive(os.path.basename(target_path))
             os.chdir(origin_dir)
-                    result['request_id'] = self.request_id
-                    self.request_id += 1
-            with open(archive_path, 'rb') as archive_fp:
-                self.translated_models.append((result['request_id'],
-                                               archive_fp.read()))
+            self.server_count += 1
         with open(
-                    os.path.join(tmpdirname, 'inference_model',
-                                 'model.pdmodel'), 'rb') as model_fp:
-                model_encoded = base64.b64encode(
-                    model_fp.read()).decode('utf-8')
+                os.path.join(target_path, 'inference_model', 'model.pdmodel'),
+                'rb') as model_fp:
+            model_encoded = base64.b64encode(model_fp.read()).decode('utf-8')
         result['pdmodel'] = model_encoded
-            if os.path.exists(archive_path):
-                os.remove(archive_path)
-
         return result
 
     @result('application/octet-stream')
     def download_model(self, request_id):
-        for stored_request_id, data in self.translated_models:
-            if str(stored_request_id) == request_id:
+        if os.path.exists(
+                os.path.join(X2PADDLE_CACHE_PATH,
+                             '{}.tar'.format(request_id))):
+            with open(
+                    os.path.join(X2PADDLE_CACHE_PATH,
+                                 '{}.tar'.format(request_id)), 'rb') as f:
+                data = f.read()
+            if self.server_count % 100 == 0:  # we check number of files every 100 request
+                file_paths = glob.glob(
+                    os.path.join(X2PADDLE_CACHE_PATH, '*.tar'))
+                if len(file_paths) >= _max_cache_numbers:
+                    file_paths = sorted(
+                        file_paths, key=os.path.getctime, reverse=True)
+                    for file_path in file_paths:
+                        try:
+                            os.remove(file_path)
+                            shutil.rmtree(
+                                os.path.join(
+                                    os.path.dirname(file_path),
+                                    os.path.splitext(
+                                        os.path.basename(file_path))[0]))
+                        except Exception:
+                            pass
             return data
 
 

visualdl/utils/dir.py

@@ -12,21 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # =======================================================================
-
-import os
 import json
-
+import os
 
 VDL_SERVER = "https://www.paddlepaddle.org.cn/paddle/visualdl/service/server"
 
-default_vdl_config = {
-    'server_url': VDL_SERVER
-}
+default_vdl_config = {'server_url': VDL_SERVER}
 
 USER_HOME = os.path.expanduser('~')
 VDL_HOME = os.path.join(USER_HOME, '.visualdl')
 CONF_HOME = os.path.join(VDL_HOME, 'conf')
 CONFIG_PATH = os.path.join(CONF_HOME, 'config.json')
+X2PADDLE_CACHE_PATH = os.path.join(VDL_HOME, 'x2paddle')
 
 
 def init_vdl_config():
@@ -35,3 +32,5 @@ def init_vdl_config():
     if not os.path.exists(CONFIG_PATH) or 0 == os.path.getsize(CONFIG_PATH):
         with open(CONFIG_PATH, 'w') as fp:
             fp.write(json.dumps(default_vdl_config))
+    if not os.path.exists(X2PADDLE_CACHE_PATH):
+        os.makedirs(X2PADDLE_CACHE_PATH, exist_ok=True)