support multiple vdl process when using gunicorn for x2paddle (#1170)

* support multiple vdl process when using gunicorn for x2paddle

* fix a bug

visualdl/component/inference/model_convert_server.py

@@ -13,10 +13,12 @@
 # limitations under the License.
 # =======================================================================
 import base64
+import glob
+import hashlib
 import json
 import os
+import shutil
 import tempfile
-from collections import deque
 from threading import Lock
 
 from flask import request
@@ -27,15 +29,18 @@ from .xarfile import archive
 from .xarfile import unarchive
 from visualdl.server.api import gen_result
 from visualdl.server.api import result
+from visualdl.utils.dir import X2PADDLE_CACHE_PATH
+
+_max_cache_numbers = 200
 
 
 class ModelConvertApi(object):
     def __init__(self):
         self.supported_formats = {'onnx', 'caffe'}
         self.lock = Lock()
-        self.translated_models = deque(
-            maxlen=5)  # used to store user's translated model for download
-        self.request_id = 0  # used to store user's request
+        self.server_count = 0  # we use this variable to count requests handled,
+        # and check the number of files every 100 requests.
+        # If more than _max_cache_numbers files in cache, we delete the last recent used 50 files.
 
     @result()
     def convert_model(self, format):
@@ -48,73 +53,101 @@ class ModelConvertApi(object):
         result['from'] = format
         result['to'] = 'paddle'
         # call x2paddle to convert models
-        with tempfile.TemporaryDirectory(
-                suffix='x2paddle_translated_models') as tmpdirname:
-            with tempfile.NamedTemporaryFile() as fp:
-                fp.write(data)
-                fp.flush()
-                try:
-                    if format == 'onnx':
-                        try:
-                            import onnx  # noqa: F401
-                        except Exception:
+        hl = hashlib.md5()
+        hl.update(data)
+        identity = hl.hexdigest()
+        result['request_id'] = identity
+        target_path = os.path.join(X2PADDLE_CACHE_PATH, identity)
+        if os.path.exists(target_path):
+            if os.path.exists(
+                    os.path.join(target_path, 'inference_model',
+                                 'model.pdmodel')):  # if data in cache
+                with open(
+                        os.path.join(target_path, 'inference_model',
+                                     'model.pdmodel'), 'rb') as model_fp:
+                    model_encoded = base64.b64encode(
+                        model_fp.read()).decode('utf-8')
+                result['pdmodel'] = model_encoded
+                return result
+        else:
+            os.makedirs(target_path, exist_ok=True)
+        with tempfile.NamedTemporaryFile() as fp:
+            fp.write(data)
+            fp.flush()
+            try:
+                if format == 'onnx':
+                    try:
+                        import onnx  # noqa: F401
+                    except Exception:
+                        raise RuntimeError(
+                            "[ERROR] onnx is not installed, use \"pip install onnx>=1.6.0\"."
+                        )
+                    onnx2paddle(fp.name, target_path)
+                elif format == 'caffe':
+                    with tempfile.TemporaryDirectory() as unarchivedir:
+                        unarchive(fp.name, unarchivedir)
+                        prototxt_path = None
+                        weight_path = None
+                        for dirname, subdirs, filenames in os.walk(
+                                unarchivedir):
+                            for filename in filenames:
+                                if '.prototxt' in filename:
+                                    prototxt_path = os.path.join(
+                                        dirname, filename)
+                                if '.caffemodel' in filename:
+                                    weight_path = os.path.join(
+                                        dirname, filename)
+                        if prototxt_path is None or weight_path is None:
                             raise RuntimeError(
-                                "[ERROR] onnx is not installed, use \"pip install onnx>=1.6.0\"."
-                            )
-                        onnx2paddle(fp.name, tmpdirname)
-                    elif format == 'caffe':
-                        with tempfile.TemporaryDirectory() as unarchivedir:
-                            unarchive(fp.name, unarchivedir)
-                            prototxt_path = None
-                            weight_path = None
-                            for dirname, subdirs, filenames in os.walk(
-                                    unarchivedir):
-                                for filename in filenames:
-                                    if '.prototxt' in filename:
-                                        prototxt_path = os.path.join(
-                                            dirname, filename)
-                                    if '.caffemodel' in filename:
-                                        weight_path = os.path.join(
-                                            dirname, filename)
-                            if prototxt_path is None or weight_path is None:
-                                raise RuntimeError(
-                                    ".prototxt or .caffemodel file is missing in your archive file, \
-                    please check files uploaded.")
-                            caffe2paddle(prototxt_path, weight_path,
-                                         tmpdirname, None)
-                except Exception as e:
-                    raise RuntimeError(
-                        "[Convertion error] {}.\n Please open an issue at \
-                            https://github.com/PaddlePaddle/X2Paddle/issues to report your problem."
-                        .format(e))
-                with self.lock:
-                    origin_dir = os.getcwd()
-                    os.chdir(os.path.dirname(tmpdirname))
-                    archive_path = os.path.join(
-                        os.path.dirname(tmpdirname),
-                        archive(os.path.basename(tmpdirname)))
-                    os.chdir(origin_dir)
-                    result['request_id'] = self.request_id
-                    self.request_id += 1
-            with open(archive_path, 'rb') as archive_fp:
-                self.translated_models.append((result['request_id'],
-                                               archive_fp.read()))
-            with open(
-                    os.path.join(tmpdirname, 'inference_model',
-                                 'model.pdmodel'), 'rb') as model_fp:
-                model_encoded = base64.b64encode(
-                    model_fp.read()).decode('utf-8')
-            result['pdmodel'] = model_encoded
-            if os.path.exists(archive_path):
-                os.remove(archive_path)
-
+                                ".prototxt or .caffemodel file is missing in your archive file, \
+                please check files uploaded.")
+                        caffe2paddle(prototxt_path, weight_path, target_path,
+                                     None)
+            except Exception as e:
+                raise RuntimeError(
+                    "[Convertion error] {}.\n Please open an issue at \
+                        https://github.com/PaddlePaddle/X2Paddle/issues to report your problem."
+                    .format(e))
+        with self.lock:  # we need to enter dirname(target_path) to archive,
+            # in case unneccessary directory added in archive.
+            origin_dir = os.getcwd()
+            os.chdir(os.path.dirname(target_path))
+            archive(os.path.basename(target_path))
+            os.chdir(origin_dir)
+            self.server_count += 1
+        with open(
+                os.path.join(target_path, 'inference_model', 'model.pdmodel'),
+                'rb') as model_fp:
+            model_encoded = base64.b64encode(model_fp.read()).decode('utf-8')
+        result['pdmodel'] = model_encoded
         return result
 
     @result('application/octet-stream')
     def download_model(self, request_id):
-        for stored_request_id, data in self.translated_models:
-            if str(stored_request_id) == request_id:
-                return data
+        if os.path.exists(
+                os.path.join(X2PADDLE_CACHE_PATH,
+                             '{}.tar'.format(request_id))):
+            with open(
+                    os.path.join(X2PADDLE_CACHE_PATH,
+                                 '{}.tar'.format(request_id)), 'rb') as f:
+                data = f.read()
+            if self.server_count % 100 == 0:  # we check number of files every 100 request
+                file_paths = glob.glob(
+                    os.path.join(X2PADDLE_CACHE_PATH, '*.tar'))
+                if len(file_paths) >= _max_cache_numbers:
+                    file_paths = sorted(
+                        file_paths, key=os.path.getctime, reverse=True)
+                    for file_path in file_paths:
+                        try:
+                            os.remove(file_path)
+                            shutil.rmtree(
+                                os.path.join(
+                                    os.path.dirname(file_path),
+                                    os.path.splitext(
+                                        os.path.basename(file_path))[0]))
+                        except Exception:
+                            pass
+            return data
 
 
 def create_model_convert_api_call():

visualdl/utils/dir.py

@@ -12,21 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # =======================================================================
-
-import os
 import json
-
+import os
 
 VDL_SERVER = "https://www.paddlepaddle.org.cn/paddle/visualdl/service/server"
 
-default_vdl_config = {
-    'server_url': VDL_SERVER
-}
+default_vdl_config = {'server_url': VDL_SERVER}
 
 USER_HOME = os.path.expanduser('~')
 VDL_HOME = os.path.join(USER_HOME, '.visualdl')
 CONF_HOME = os.path.join(VDL_HOME, 'conf')
 CONFIG_PATH = os.path.join(CONF_HOME, 'config.json')
+X2PADDLE_CACHE_PATH = os.path.join(VDL_HOME, 'x2paddle')
 
 
 def init_vdl_config():
@@ -35,3 +32,5 @@ def init_vdl_config():
     if not os.path.exists(CONFIG_PATH) or 0 == os.path.getsize(CONFIG_PATH):
         with open(CONFIG_PATH, 'w') as fp:
             fp.write(json.dumps(default_vdl_config))
+    if not os.path.exists(X2PADDLE_CACHE_PATH):
+        os.makedirs(X2PADDLE_CACHE_PATH, exist_ok=True)