未验证 提交 8b7e9496 编写于 作者: C chenjian 提交者: GitHub

support multiple vdl process when using gunicorn for x2paddle (#1170)

* support multiple vdl process when using gunicorn for x2paddle

* fix a bug
上级 9a16e460
......@@ -13,10 +13,12 @@
# limitations under the License.
# =======================================================================
import base64
import glob
import hashlib
import json
import os
import shutil
import tempfile
from collections import deque
from threading import Lock
from flask import request
......@@ -27,15 +29,18 @@ from .xarfile import archive
from .xarfile import unarchive
from visualdl.server.api import gen_result
from visualdl.server.api import result
from visualdl.utils.dir import X2PADDLE_CACHE_PATH
_max_cache_numbers = 200
class ModelConvertApi(object):
def __init__(self):
self.supported_formats = {'onnx', 'caffe'}
self.lock = Lock()
self.translated_models = deque(
maxlen=5) # used to store user's translated model for download
self.request_id = 0 # used to store user's request
self.server_count = 0 # we use this variable to count requests handled,
# and check the number of files every 100 requests.
# If more than _max_cache_numbers files in cache, we delete the last recent used 50 files.
@result()
def convert_model(self, format):
......@@ -48,8 +53,24 @@ class ModelConvertApi(object):
result['from'] = format
result['to'] = 'paddle'
# call x2paddle to convert models
with tempfile.TemporaryDirectory(
suffix='x2paddle_translated_models') as tmpdirname:
hl = hashlib.md5()
hl.update(data)
identity = hl.hexdigest()
result['request_id'] = identity
target_path = os.path.join(X2PADDLE_CACHE_PATH, identity)
if os.path.exists(target_path):
if os.path.exists(
os.path.join(target_path, 'inference_model',
'model.pdmodel')): # if data in cache
with open(
os.path.join(target_path, 'inference_model',
'model.pdmodel'), 'rb') as model_fp:
model_encoded = base64.b64encode(
model_fp.read()).decode('utf-8')
result['pdmodel'] = model_encoded
return result
else:
os.makedirs(target_path, exist_ok=True)
with tempfile.NamedTemporaryFile() as fp:
fp.write(data)
fp.flush()
......@@ -61,7 +82,7 @@ class ModelConvertApi(object):
raise RuntimeError(
"[ERROR] onnx is not installed, use \"pip install onnx>=1.6.0\"."
)
onnx2paddle(fp.name, tmpdirname)
onnx2paddle(fp.name, target_path)
elif format == 'caffe':
with tempfile.TemporaryDirectory() as unarchivedir:
unarchive(fp.name, unarchivedir)
......@@ -80,40 +101,52 @@ class ModelConvertApi(object):
raise RuntimeError(
".prototxt or .caffemodel file is missing in your archive file, \
please check files uploaded.")
caffe2paddle(prototxt_path, weight_path,
tmpdirname, None)
caffe2paddle(prototxt_path, weight_path, target_path,
None)
except Exception as e:
raise RuntimeError(
"[Convertion error] {}.\n Please open an issue at \
https://github.com/PaddlePaddle/X2Paddle/issues to report your problem."
.format(e))
with self.lock:
with self.lock: # we need to enter dirname(target_path) to archive,
# in case unneccessary directory added in archive.
origin_dir = os.getcwd()
os.chdir(os.path.dirname(tmpdirname))
archive_path = os.path.join(
os.path.dirname(tmpdirname),
archive(os.path.basename(tmpdirname)))
os.chdir(os.path.dirname(target_path))
archive(os.path.basename(target_path))
os.chdir(origin_dir)
result['request_id'] = self.request_id
self.request_id += 1
with open(archive_path, 'rb') as archive_fp:
self.translated_models.append((result['request_id'],
archive_fp.read()))
self.server_count += 1
with open(
os.path.join(tmpdirname, 'inference_model',
'model.pdmodel'), 'rb') as model_fp:
model_encoded = base64.b64encode(
model_fp.read()).decode('utf-8')
os.path.join(target_path, 'inference_model', 'model.pdmodel'),
'rb') as model_fp:
model_encoded = base64.b64encode(model_fp.read()).decode('utf-8')
result['pdmodel'] = model_encoded
if os.path.exists(archive_path):
os.remove(archive_path)
return result
@result('application/octet-stream')
def download_model(self, request_id):
for stored_request_id, data in self.translated_models:
if str(stored_request_id) == request_id:
if os.path.exists(
os.path.join(X2PADDLE_CACHE_PATH,
'{}.tar'.format(request_id))):
with open(
os.path.join(X2PADDLE_CACHE_PATH,
'{}.tar'.format(request_id)), 'rb') as f:
data = f.read()
if self.server_count % 100 == 0: # we check number of files every 100 request
file_paths = glob.glob(
os.path.join(X2PADDLE_CACHE_PATH, '*.tar'))
if len(file_paths) >= _max_cache_numbers:
file_paths = sorted(
file_paths, key=os.path.getctime, reverse=True)
for file_path in file_paths:
try:
os.remove(file_path)
shutil.rmtree(
os.path.join(
os.path.dirname(file_path),
os.path.splitext(
os.path.basename(file_path))[0]))
except Exception:
pass
return data
......
......@@ -12,21 +12,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# =======================================================================
import os
import json
import os
VDL_SERVER = "https://www.paddlepaddle.org.cn/paddle/visualdl/service/server"
default_vdl_config = {
'server_url': VDL_SERVER
}
default_vdl_config = {'server_url': VDL_SERVER}
USER_HOME = os.path.expanduser('~')
VDL_HOME = os.path.join(USER_HOME, '.visualdl')
CONF_HOME = os.path.join(VDL_HOME, 'conf')
CONFIG_PATH = os.path.join(CONF_HOME, 'config.json')
X2PADDLE_CACHE_PATH = os.path.join(VDL_HOME, 'x2paddle')
def init_vdl_config():
......@@ -35,3 +32,5 @@ def init_vdl_config():
if not os.path.exists(CONFIG_PATH) or 0 == os.path.getsize(CONFIG_PATH):
with open(CONFIG_PATH, 'w') as fp:
fp.write(json.dumps(default_vdl_config))
if not os.path.exists(X2PADDLE_CACHE_PATH):
os.makedirs(X2PADDLE_CACHE_PATH, exist_ok=True)
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册