提交 0f7b5506 编写于 作者: 走神的阿圆's avatar 走神的阿圆

Add serving stop

上级 d3d38972
...@@ -25,6 +25,7 @@ from paddlehub.autofinetune.autoft import PSHE2 ...@@ -25,6 +25,7 @@ from paddlehub.autofinetune.autoft import PSHE2
from paddlehub.autofinetune.autoft import HAZero from paddlehub.autofinetune.autoft import HAZero
from paddlehub.autofinetune.evaluator import FullTrailEvaluator from paddlehub.autofinetune.evaluator import FullTrailEvaluator
from paddlehub.autofinetune.evaluator import PopulationBasedEvaluator from paddlehub.autofinetune.evaluator import PopulationBasedEvaluator
from paddlehub.common.hub_server import CacheUpdater
class AutoFineTuneCommand(BaseCommand): class AutoFineTuneCommand(BaseCommand):
...@@ -96,6 +97,7 @@ class AutoFineTuneCommand(BaseCommand): ...@@ -96,6 +97,7 @@ class AutoFineTuneCommand(BaseCommand):
return options_str return options_str
def execute(self, argv): def execute(self, argv):
CacheUpdater("hub_autofinetune").start()
if not argv: if not argv:
print("ERROR: Please specify a script to be finetuned in python.\n") print("ERROR: Please specify a script to be finetuned in python.\n")
self.help() self.help()
......
...@@ -21,6 +21,7 @@ import os ...@@ -21,6 +21,7 @@ import os
from paddlehub.commands.base_command import BaseCommand from paddlehub.commands.base_command import BaseCommand
from paddlehub.common.dir import CACHE_HOME from paddlehub.common.dir import CACHE_HOME
from paddlehub.common.hub_server import CacheUpdater
def file_num_in_dir(dirname): def file_num_in_dir(dirname):
...@@ -56,6 +57,7 @@ class ClearCommand(BaseCommand): ...@@ -56,6 +57,7 @@ class ClearCommand(BaseCommand):
return CACHE_HOME return CACHE_HOME
def execute(self, argv): def execute(self, argv):
CacheUpdater("hub_clear").start()
result = True result = True
total_file_size = 0 total_file_size = 0
total_file_count = 0 total_file_count = 0
......
...@@ -25,6 +25,10 @@ import re ...@@ -25,6 +25,10 @@ import re
from paddlehub.commands.base_command import BaseCommand, ENTRY from paddlehub.commands.base_command import BaseCommand, ENTRY
from paddlehub.common.dir import CONF_HOME from paddlehub.common.dir import CONF_HOME
from paddlehub.common.server_config import default_server_config from paddlehub.common.server_config import default_server_config
from paddlehub.common.hub_server import CacheUpdater
from paddlehub.common.hub_server import HubServer
HubServer()
class ConfigCommand(BaseCommand): class ConfigCommand(BaseCommand):
...@@ -98,6 +102,7 @@ class ConfigCommand(BaseCommand): ...@@ -98,6 +102,7 @@ class ConfigCommand(BaseCommand):
print(str) print(str)
def execute(self, argv): def execute(self, argv):
CacheUpdater("hub_config").start()
args = self.parser.parse_args() args = self.parser.parse_args()
if args.option is None: if args.option is None:
ConfigCommand.show_config() ConfigCommand.show_config()
......
...@@ -24,6 +24,7 @@ import paddlehub as hub ...@@ -24,6 +24,7 @@ import paddlehub as hub
from paddlehub.common import utils from paddlehub.common import utils
from paddlehub.common.downloader import default_downloader from paddlehub.common.downloader import default_downloader
from paddlehub.commands.base_command import BaseCommand, ENTRY from paddlehub.commands.base_command import BaseCommand, ENTRY
from paddlehub.common.hub_server import CacheUpdater
class DownloadCommand(BaseCommand): class DownloadCommand(BaseCommand):
...@@ -52,6 +53,7 @@ class DownloadCommand(BaseCommand): ...@@ -52,6 +53,7 @@ class DownloadCommand(BaseCommand):
mod_name = argv[0] mod_name = argv[0]
mod_version = None if "==" not in mod_name else mod_name.split("==")[1] mod_version = None if "==" not in mod_name else mod_name.split("==")[1]
mod_name = mod_name if "==" not in mod_name else mod_name.split("==")[0] mod_name = mod_name if "==" not in mod_name else mod_name.split("==")[0]
CacheUpdater("hub_download", mod_name, mod_version).start()
self.args = self.parser.parse_args(argv[1:]) self.args = self.parser.parse_args(argv[1:])
self.args.type = self.check_type(self.args.type) self.args.type = self.check_type(self.args.type)
......
...@@ -18,6 +18,7 @@ from __future__ import division ...@@ -18,6 +18,7 @@ from __future__ import division
from __future__ import print_function from __future__ import print_function
from paddlehub.commands.base_command import BaseCommand from paddlehub.commands.base_command import BaseCommand
from paddlehub.common.hub_server import CacheUpdater
class HelpCommand(BaseCommand): class HelpCommand(BaseCommand):
...@@ -32,6 +33,7 @@ class HelpCommand(BaseCommand): ...@@ -32,6 +33,7 @@ class HelpCommand(BaseCommand):
return BaseCommand.command_dict return BaseCommand.command_dict
def execute(self, argv): def execute(self, argv):
CacheUpdater("hub_help").start()
hub_command = BaseCommand.command_dict["hub"] hub_command = BaseCommand.command_dict["hub"]
help_text = "\n" help_text = "\n"
help_text += "Usage:\n" help_text += "Usage:\n"
......
...@@ -23,6 +23,7 @@ import os ...@@ -23,6 +23,7 @@ import os
from paddlehub.common import utils from paddlehub.common import utils
from paddlehub.module.manager import default_module_manager from paddlehub.module.manager import default_module_manager
from paddlehub.commands.base_command import BaseCommand, ENTRY from paddlehub.commands.base_command import BaseCommand, ENTRY
from paddlehub.common.hub_server import CacheUpdater
class InstallCommand(BaseCommand): class InstallCommand(BaseCommand):
...@@ -57,6 +58,7 @@ class InstallCommand(BaseCommand): ...@@ -57,6 +58,7 @@ class InstallCommand(BaseCommand):
"==")[1] "==")[1]
module_name = module_name if "==" not in module_name else module_name.split( module_name = module_name if "==" not in module_name else module_name.split(
"==")[0] "==")[0]
CacheUpdater("hub_install", module_name, module_version).start()
result, tips, module_dir = default_module_manager.install_module( result, tips, module_dir = default_module_manager.install_module(
module_name=module_name, module_name=module_name,
module_version=module_version, module_version=module_version,
......
...@@ -22,6 +22,7 @@ from paddlehub.common.downloader import default_downloader ...@@ -22,6 +22,7 @@ from paddlehub.common.downloader import default_downloader
from paddlehub.module.manager import default_module_manager from paddlehub.module.manager import default_module_manager
from paddlehub.commands.base_command import BaseCommand from paddlehub.commands.base_command import BaseCommand
from paddlehub.common.cml_utils import TablePrinter from paddlehub.common.cml_utils import TablePrinter
from paddlehub.common.hub_server import CacheUpdater
class ListCommand(BaseCommand): class ListCommand(BaseCommand):
...@@ -33,6 +34,7 @@ class ListCommand(BaseCommand): ...@@ -33,6 +34,7 @@ class ListCommand(BaseCommand):
self.description = "List all installed PaddleHub modules." self.description = "List all installed PaddleHub modules."
def execute(self, argv): def execute(self, argv):
CacheUpdater("hub_list").start()
all_modules = default_module_manager.all_modules() all_modules = default_module_manager.all_modules()
if utils.is_windows(): if utils.is_windows():
placeholders = [20, 40] placeholders = [20, 40]
......
...@@ -30,6 +30,7 @@ import cv2 ...@@ -30,6 +30,7 @@ import cv2
from paddlehub.commands.base_command import BaseCommand, ENTRY from paddlehub.commands.base_command import BaseCommand, ENTRY
from paddlehub.io.parser import yaml_parser, txt_parser from paddlehub.io.parser import yaml_parser, txt_parser
from paddlehub.module.manager import default_module_manager from paddlehub.module.manager import default_module_manager
from paddlehub.common.hub_server import CacheUpdater
import paddlehub as hub import paddlehub as hub
...@@ -212,13 +213,14 @@ class RunCommand(BaseCommand): ...@@ -212,13 +213,14 @@ class RunCommand(BaseCommand):
raise RuntimeError("ERROR: Format of %s is illegal." % file_path) raise RuntimeError("ERROR: Format of %s is illegal." % file_path)
def execute(self, argv): def execute(self, argv):
if not argv: if not argv:
print("ERROR: Please specify a module name.\n") print("ERROR: Please specify a module name.\n")
self.help() self.help()
return False return False
module_name = argv[0] module_name = argv[0]
CacheUpdater("hub_run", module_name).start()
self.parser.prog = '%s %s %s' % (ENTRY, self.name, module_name) self.parser.prog = '%s %s %s' % (ENTRY, self.name, module_name)
self.arg_input_group = self.parser.add_argument_group( self.arg_input_group = self.parser.add_argument_group(
title="Input options", description="Data input to the module") title="Input options", description="Data input to the module")
......
...@@ -23,6 +23,7 @@ import paddlehub as hub ...@@ -23,6 +23,7 @@ import paddlehub as hub
from paddlehub.common import utils from paddlehub.common import utils
from paddlehub.commands.base_command import BaseCommand, ENTRY from paddlehub.commands.base_command import BaseCommand, ENTRY
from paddlehub.common.cml_utils import TablePrinter from paddlehub.common.cml_utils import TablePrinter
from paddlehub.common.hub_server import CacheUpdater
class SearchCommand(BaseCommand): class SearchCommand(BaseCommand):
...@@ -43,6 +44,7 @@ class SearchCommand(BaseCommand): ...@@ -43,6 +44,7 @@ class SearchCommand(BaseCommand):
argv = ['.*'] argv = ['.*']
resource_name = argv[0] resource_name = argv[0]
CacheUpdater("hub_search", resource_name).start()
extra = {"command": "search"} extra = {"command": "search"}
resource_list = hub.HubServer().search_resource( resource_list = hub.HubServer().search_resource(
resource_name, resource_type="Module", extra=extra) resource_name, resource_type="Module", extra=extra)
......
...@@ -25,31 +25,56 @@ import json ...@@ -25,31 +25,56 @@ import json
import paddlehub as hub import paddlehub as hub
from paddlehub.commands.base_command import BaseCommand, ENTRY from paddlehub.commands.base_command import BaseCommand, ENTRY
from paddlehub.serving import app_single as app from paddlehub.serving import app_single as app
from paddlehub.common.dir import CONF_HOME
from paddlehub.common.hub_server import CacheUpdater
import multiprocessing import multiprocessing
import gunicorn.app.base import time
import signal
if platform.system() == "Windows":
def number_of_workers(): class StandaloneApplication(object):
return (multiprocessing.cpu_count() * 2) + 1 def __init__(self):
pass
def load_config(self):
pass
class StandaloneApplication(gunicorn.app.base.BaseApplication): def load(self):
def __init__(self, app, options=None): pass
self.options = options or {} else:
self.application = app import gunicorn.app.base
super(StandaloneApplication, self).__init__()
def load_config(self): class StandaloneApplication(gunicorn.app.base.BaseApplication):
config = { def __init__(self, app, options=None):
key: value self.options = options or {}
for key, value in self.options.items() self.application = app
if key in self.cfg.settings and value is not None super(StandaloneApplication, self).__init__()
}
for key, value in config.items(): def load_config(self):
self.cfg.set(key.lower(), value) config = {
key: value
for key, value in self.options.items()
if key in self.cfg.settings and value is not None
}
for key, value in config.items():
self.cfg.set(key.lower(), value)
def load(self):
return self.application
def number_of_workers():
return (multiprocessing.cpu_count() * 2) + 1
def load(self):
return self.application def pid_is_exist(pid):
try:
os.kill(pid, 0)
except:
return False
else:
return True
class ServingCommand(BaseCommand): class ServingCommand(BaseCommand):
...@@ -79,6 +104,59 @@ class ServingCommand(BaseCommand): ...@@ -79,6 +104,59 @@ class ServingCommand(BaseCommand):
self.parser.add_argument("--port", "-p", nargs="?", default=8866) self.parser.add_argument("--port", "-p", nargs="?", default=8866)
self.parser.add_argument("--gpu", "-i", nargs="?", default=0) self.parser.add_argument("--gpu", "-i", nargs="?", default=0)
def dump_pid_file(self):
pid = os.getpid()
filepath = os.path.join(CONF_HOME,
"serving_" + self.args.port + ".json")
if os.path.exists(filepath):
os.remove(filepath)
with open(filepath, "w") as fp:
info = {
"pid": pid,
"module": self.args.modules,
"start_time": time.time()
}
json.dump(info, fp)
@staticmethod
def load_pid_file(filepath, port=None):
if port is None:
port = os.path.basename(filepath).split(".")[0].split("_")[1]
if not os.path.exists(filepath):
print(
"PaddleHub-Serving config file is not exists, "
"please confirm the port [%s] you specified is correct." % port)
return False
with open(filepath, "r") as fp:
info = json.load(fp)
return info
def stop_serving(self, port):
filepath = os.path.join(CONF_HOME, "serving_" + str(port) + ".json")
info = self.load_pid_file(filepath, port)
if info is False:
return
pid = info["pid"]
module = info["module"]
start_time = info["start_time"]
if os.path.exists(filepath):
os.remove(filepath)
if not pid_is_exist(pid):
print("PaddleHub-Serving has been stopped.")
return
print("PaddleHub-Serving will stop.")
CacheUpdater(
"hub_serving_stop",
module=module,
addition={
"period_time": time.time() - start_time
}).start()
if platform.system() == "Windows":
os.kill(pid, signal.SIGTERM)
else:
os.killpg(pid, signal.SIGTERM)
@staticmethod @staticmethod
def start_bert_serving(args): def start_bert_serving(args):
if platform.system() != "Linux": if platform.system() != "Linux":
...@@ -120,6 +198,10 @@ class ServingCommand(BaseCommand): ...@@ -120,6 +198,10 @@ class ServingCommand(BaseCommand):
configs.pop() configs.pop()
module_exist.update({module_name: module_version}) module_exist.update({module_name: module_version})
try: try:
CacheUpdater(
"hub_serving_start",
module=module_name,
version=module_version).start()
m = hub.Module(name=module_name, version=module_version) m = hub.Module(name=module_name, version=module_version)
method_name = m.desc.attr.map.data['default_signature'].s method_name = m.desc.attr.map.data['default_signature'].s
if method_name == "": if method_name == "":
...@@ -135,6 +217,28 @@ class ServingCommand(BaseCommand): ...@@ -135,6 +217,28 @@ class ServingCommand(BaseCommand):
exit(1) exit(1)
return configs return configs
def start_app_with_file(self, configs, workers):
port = configs.get("port", 8866)
if ServingCommand.is_port_occupied("127.0.0.1", port) is True:
print("Port %s is occupied, please change it." % port)
return False
modules = configs.get("modules_info")
module = [str(i["module"]) + "==" + str(i["version"]) for i in modules]
module_info = ServingCommand.preinstall_modules(module)
for index in range(len(module_info)):
modules[index].update(module_info[index])
options = {
"bind": "0.0.0.0:%s" % port,
"workers": workers,
"pid": "./pid.txt"
}
configs["modules_info"] = modules
self.dump_pid_file()
StandaloneApplication(
app.create_app(init_flag=False, configs=configs), options).run()
@staticmethod @staticmethod
def start_single_app_with_file(configs): def start_single_app_with_file(configs):
use_gpu = configs.get("use_gpu", False) use_gpu = configs.get("use_gpu", False)
...@@ -161,6 +265,30 @@ class ServingCommand(BaseCommand): ...@@ -161,6 +265,30 @@ class ServingCommand(BaseCommand):
app.create_app(init_flag=False, configs=configs), options).run() app.create_app(init_flag=False, configs=configs), options).run()
print("PaddleHub-Serving has been stopped.") print("PaddleHub-Serving has been stopped.")
def start_app_with_args(self, workers):
module = self.args.modules
if module is not None:
use_gpu = self.args.use_gpu
port = self.args.port
if ServingCommand.is_port_occupied("127.0.0.1", port) is True:
print("Port %s is occupied, please change it." % port)
return False
module_info = ServingCommand.preinstall_modules(module)
[
item.update({
"batch_size": 1,
"queue_size": 20
}) for item in module_info
]
options = {"bind": "0.0.0.0:%s" % port, "workers": workers}
configs = {"use_gpu": use_gpu, "modules_info": module_info}
self.dump_pid_file()
StandaloneApplication(
app.create_app(init_flag=False, configs=configs),
options).run()
else:
print("Lack of necessary parameters!")
def start_single_app_with_args(self): def start_single_app_with_args(self):
module = self.args.modules module = self.args.modules
if module is not None: if module is not None:
...@@ -212,52 +340,62 @@ class ServingCommand(BaseCommand): ...@@ -212,52 +340,62 @@ class ServingCommand(BaseCommand):
with open(config_file, "r") as fp: with open(config_file, "r") as fp:
configs = json.load(fp) configs = json.load(fp)
use_multiprocess = configs.get("use_multiprocess", False) use_multiprocess = configs.get("use_multiprocess", False)
if use_multiprocess is True: if platform.system() == "Windows":
if platform.system() == "Windows": print(
print( "Warning: Windows cannot use multiprocess working "
"Warning: Windows cannot use multiprocess working " "mode, Hub-Serving will switch to single process mode"
"mode, Hub-Serving will switch to single process mode" )
)
ServingCommand.start_single_app_with_file(configs)
else:
ServingCommand.start_multi_app_with_file(configs)
else:
ServingCommand.start_single_app_with_file(configs) ServingCommand.start_single_app_with_file(configs)
else:
if use_multiprocess is True:
self.start_app_with_file(configs,
number_of_workers())
else:
self.start_app_with_file(configs, 1)
else: else:
print("config_file ", config_file, "not exists.") print("config_file ", config_file, "not exists.")
else: else:
if self.args.use_multiprocess is True: if platform.system() == "Windows":
if platform.system() == "Windows": print("Warning: Windows cannot use multiprocess working "
print( "mode, Hub-Serving will switch to single process mode")
"Warning: Windows cannot use multiprocess working "
"mode, Hub-Serving will switch to single process mode")
self.start_single_app_with_args()
else:
self.start_multi_app_with_args()
else:
self.start_single_app_with_args() self.start_single_app_with_args()
else:
if self.args.use_multiprocess is True:
self.start_app_with_args(number_of_workers())
else:
self.start_app_with_args(1)
@staticmethod @staticmethod
def show_help(): def show_help():
str = "serving <option>\n" str = "serving <option>\n"
str += "\tManage PaddleHub-Serving.\n" str += "\tManage PaddleHub-Serving.\n"
str += "sub command:\n" str += "sub command:\n"
str += "start\n" str += "1. start\n"
str += "\tStart PaddleHub-Serving if specifies this parameter.\n" str += "\tStart PaddleHub-Serving if specifies this parameter.\n"
str += "start bert_service\n" str += "2. start bert_service\n"
str += "\tStart Bert Service if specifies this parameter.\n" str += "\tStart Bert Service if specifies this parameter.\n"
str += "option:\n" str += "[start] option:\n"
str += "--modules/-m [module1==version, module2==version...]\n" str += "--modules/-m [module1==version, module2==version...]\n"
str += "\tPre-install modules via this parameter list.\n" str += "\tPre-install modules via this parameter list.\n"
str += "--port/-p XXXX\n" str += "--port/-p XXXX\n"
str += "\tUse port XXXX for serving.\n" str += "\tUse port XXXX for serving.\n"
str += "--use_gpu\n" str += "--use_gpu\n"
str += "\tUse gpu for predicting if specifies this parameter.\n" str += "\tUse gpu for predicting if specifies this parameter.\n"
str += "--gpu\n" str += "--use_multiprocess\n"
str += "\tSpecify the graphics card to use, only work for Bert as Service\n" str += "\tChoose multoprocess mode, cannot be use on Windows.\n"
str += "--config/-c file_path\n" str += "--config/-c file_path\n"
str += "\tUse configs in file to starting paddlehub serving." str += "\tUse configs in file to starting paddlehub serving. "
str += "Other parameter will be ignored if specifies this parameter.\n" str += "Other parameter will be ignored if specifies this parameter.\n"
str += "[start bert_service] option:\n"
str += "--modules/-m\n"
str += "\tPre-install modules via this parameter.\n"
str += "--port/-p XXXX\n"
str += "\tUse port XXXX for serving.\n"
str += "--use_gpu\n"
str += "\tUse gpu for predicting if specifies this parameter.\n"
str += "--gpu\n"
str += "\tSpecify the graphics card to use.\n"
print(str) print(str)
def execute(self, argv): def execute(self, argv):
...@@ -273,6 +411,11 @@ class ServingCommand(BaseCommand): ...@@ -273,6 +411,11 @@ class ServingCommand(BaseCommand):
self.start_serving() self.start_serving()
else: else:
ServingCommand.show_help() ServingCommand.show_help()
elif self.args.sub_command == "stop":
if self.args.bert_service == "bert_service":
print("Please stop bert_service by kill process by yourself")
elif self.args.bert_service is None:
self.stop_serving(port=self.args.port)
else: else:
ServingCommand.show_help() ServingCommand.show_help()
......
...@@ -26,6 +26,7 @@ from paddlehub.common.cml_utils import TablePrinter ...@@ -26,6 +26,7 @@ from paddlehub.common.cml_utils import TablePrinter
from paddlehub.module.manager import default_module_manager from paddlehub.module.manager import default_module_manager
from paddlehub.module.module import Module from paddlehub.module.module import Module
from paddlehub.io.parser import yaml_parser from paddlehub.io.parser import yaml_parser
from paddlehub.common.hub_server import CacheUpdater
class ShowCommand(BaseCommand): class ShowCommand(BaseCommand):
...@@ -116,6 +117,7 @@ class ShowCommand(BaseCommand): ...@@ -116,6 +117,7 @@ class ShowCommand(BaseCommand):
return False return False
module_name = argv[0] module_name = argv[0]
CacheUpdater("hub_show", module_name).start()
# nlp model # nlp model
model_info_file = os.path.join(module_name, "info.yml") model_info_file = os.path.join(module_name, "info.yml")
......
...@@ -22,6 +22,7 @@ import argparse ...@@ -22,6 +22,7 @@ import argparse
from paddlehub.common import utils from paddlehub.common import utils
from paddlehub.module.manager import default_module_manager from paddlehub.module.manager import default_module_manager
from paddlehub.commands.base_command import BaseCommand, ENTRY from paddlehub.commands.base_command import BaseCommand, ENTRY
from paddlehub.common.hub_server import CacheUpdater
class UninstallCommand(BaseCommand): class UninstallCommand(BaseCommand):
...@@ -43,6 +44,7 @@ class UninstallCommand(BaseCommand): ...@@ -43,6 +44,7 @@ class UninstallCommand(BaseCommand):
self.help() self.help()
return False return False
module_name = argv[0] module_name = argv[0]
CacheUpdater("hub_uninstall", module_name).start()
result, tips = default_module_manager.uninstall_module( result, tips = default_module_manager.uninstall_module(
module_name=module_name) module_name=module_name)
print(tips) print(tips)
......
...@@ -19,6 +19,7 @@ from __future__ import print_function ...@@ -19,6 +19,7 @@ from __future__ import print_function
from paddlehub import version from paddlehub import version
from paddlehub.commands.base_command import BaseCommand from paddlehub.commands.base_command import BaseCommand
from paddlehub.common.hub_server import CacheUpdater
class VersionCommand(BaseCommand): class VersionCommand(BaseCommand):
...@@ -30,6 +31,7 @@ class VersionCommand(BaseCommand): ...@@ -30,6 +31,7 @@ class VersionCommand(BaseCommand):
self.description = "Show PaddleHub's version." self.description = "Show PaddleHub's version."
def execute(self, argv): def execute(self, argv):
CacheUpdater("hub_version").start()
print("hub %s" % version.hub_version) print("hub %s" % version.hub_version)
return True return True
......
#coding:utf-8
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import threading
def synchronized(func):
func.__lock__ = threading.Lock()
def synced_func(*args, **kwargs):
with func.__lock__:
return func(*args, **kwargs)
return synced_func
def singleton(cls):
_instance = {}
@synchronized
def _get_instance(*args, **kwargs):
if cls not in _instance:
_instance[cls] = cls(*args, **kwargs)
return _instance[cls]
return _get_instance
...@@ -28,37 +28,17 @@ import threading ...@@ -28,37 +28,17 @@ import threading
from paddlehub.common import utils, srv_utils from paddlehub.common import utils, srv_utils
from paddlehub.common.downloader import default_downloader from paddlehub.common.downloader import default_downloader
from paddlehub.common.decorator_utils import singleton
from paddlehub.common.server_config import default_server_config from paddlehub.common.server_config import default_server_config
from paddlehub.io.parser import yaml_parser from paddlehub.io.parser import yaml_parser
from paddlehub.common.lock import lock from paddlehub.common.lock import lock
from paddlehub.common.dir import CONF_HOME, CACHE_HOME from paddlehub.common.dir import CONF_HOME, CACHE_HOME
from paddlehub.common.utils import ConfigInfo
RESOURCE_LIST_FILE = "resource_list_file.yml" RESOURCE_LIST_FILE = "resource_list_file.yml"
CACHE_TIME = 60 * 10 CACHE_TIME = 60 * 10
def synchronized(func):
func.__lock__ = threading.Lock()
def synced_func(*args, **kwargs):
with func.__lock__:
return func(*args, **kwargs)
return synced_func
def singleton(cls):
_instance = {}
@synchronized
def _get_instance(*args, **kwargs):
if cls not in _instance:
_instance[cls] = cls(*args, **kwargs)
return _instance[cls]
return _get_instance
@singleton @singleton
class HubServer(object): class HubServer(object):
def __init__(self, config_file_path=None): def __init__(self, config_file_path=None):
...@@ -72,8 +52,9 @@ class HubServer(object): ...@@ -72,8 +52,9 @@ class HubServer(object):
fp.write(json.dumps(default_server_config)) fp.write(json.dumps(default_server_config))
lock.flock(fp, lock.LOCK_UN) lock.flock(fp, lock.LOCK_UN)
with open(config_file_path) as fp: with open(config_file_path, "r") as fp:
self.config = json.load(fp) self.config = json.load(fp)
fp_lock = open(config_file_path) fp_lock = open(config_file_path)
lock.flock(fp_lock, lock.LOCK_EX) lock.flock(fp_lock, lock.LOCK_EX)
...@@ -308,29 +289,44 @@ class HubServer(object): ...@@ -308,29 +289,44 @@ class HubServer(object):
class CacheUpdater(threading.Thread): class CacheUpdater(threading.Thread):
def __init__(self, module, version=None): def __init__(self,
command="update_cache",
module=None,
version=None,
addition=None):
threading.Thread.__init__(self) threading.Thread.__init__(self)
self.command = command
self.module = module self.module = module
self.version = version self.version = version
self.addition = addition
def update_resource_list_file(self, module, version=None): def update_resource_list_file(self,
command="update_cache",
module=None,
version=None,
addition=None):
payload = {'word': module} payload = {'word': module}
if version: if version:
payload['version'] = version payload['version'] = version
api_url = srv_utils.uri_path(HubServer().get_server_url(), 'search') api_url = srv_utils.uri_path(HubServer().get_server_url(), 'search')
cache_path = os.path.join(CACHE_HOME, RESOURCE_LIST_FILE) cache_path = os.path.join(CACHE_HOME, RESOURCE_LIST_FILE)
hub_name = ConfigInfo().get_hub_name()
if os.path.exists(cache_path): if os.path.exists(cache_path):
extra = { extra = {
"command": "update_cache", "command": command,
"mtime": os.stat(cache_path).st_mtime "mtime": os.stat(cache_path).st_mtime,
"hub_name": hub_name
} }
else: else:
extra = { extra = {
"command": "update_cache", "command": command,
"mtime": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) "mtime": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
"hub_name": hub_name
} }
if addition is not None:
extra.update({"addition": addition})
try: try:
r = srv_utils.hub_request(api_url, payload, extra) r = srv_utils.hub_request(api_url, payload, extra, timeout=0.1)
if r.get("update_cache", 0) == 1: if r.get("update_cache", 0) == 1:
with open(cache_path, 'w+') as fp: with open(cache_path, 'w+') as fp:
yaml.safe_dump({'resource_list': r['data']}, fp) yaml.safe_dump({'resource_list': r['data']}, fp)
...@@ -338,7 +334,8 @@ class CacheUpdater(threading.Thread): ...@@ -338,7 +334,8 @@ class CacheUpdater(threading.Thread):
pass pass
def run(self): def run(self):
self.update_resource_list_file(self.module, self.version) self.update_resource_list_file(self.command, self.module, self.version,
self.addition)
def server_check(): def server_check():
......
...@@ -11,6 +11,7 @@ ...@@ -11,6 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from paddlehub.common.utils import ConfigInfo
HUB_SERVERS = ["http://paddlepaddle.org.cn/paddlehub"] HUB_SERVERS = ["http://paddlepaddle.org.cn/paddlehub"]
...@@ -18,5 +19,6 @@ default_server_config = { ...@@ -18,5 +19,6 @@ default_server_config = {
"server_url": HUB_SERVERS, "server_url": HUB_SERVERS,
"resource_storage_server_url": "https://bj.bcebos.com/paddlehub-data/", "resource_storage_server_url": "https://bj.bcebos.com/paddlehub-data/",
"debug": False, "debug": False,
"log_level": "DEBUG" "log_level": "DEBUG",
"hub_name": ConfigInfo().get_hub_name()
} }
...@@ -31,9 +31,9 @@ def uri_path(server_url, api): ...@@ -31,9 +31,9 @@ def uri_path(server_url, api):
return srv return srv
def hub_request(api, params, extra=None): def hub_request(api, params, extra=None, timeout=8):
params['hub_version'] = version.hub_version params['hub_version'] = version.hub_version
params['paddle_version'] = paddle.__version__ params['paddle_version'] = paddle.__version__
params["extra"] = json.dumps(extra) params["extra"] = json.dumps(extra)
r = requests.get(api, params) r = requests.get(api, params, timeout=timeout)
return r.json() return r.json()
...@@ -22,12 +22,16 @@ import os ...@@ -22,12 +22,16 @@ import os
import multiprocessing import multiprocessing
import hashlib import hashlib
import platform import platform
import uuid
import json
import paddle.fluid as fluid import paddle.fluid as fluid
import six import six
from paddlehub.module import module_desc_pb2 from paddlehub.module import module_desc_pb2
from paddlehub.common.logger import logger from paddlehub.common.logger import logger
from paddlehub.common.dir import CONF_HOME
from paddlehub.common.decorator_utils import singleton
def version_compare(version1, version2): def version_compare(version1, version2):
...@@ -55,6 +59,24 @@ def get_platform(): ...@@ -55,6 +59,24 @@ def get_platform():
return platform.platform() return platform.platform()
@singleton
class ConfigInfo(object):
def __init__(self):
self.filepath = os.path.join(CONF_HOME, "config.json")
self.hub_name = None
self.configs = None
if os.path.exists(self.filepath):
with open(self.filepath, "r") as fp:
self.configs = json.load(fp)
self.use_id = self.configs.get("hub_name", None)
def get_hub_name(self):
if self.hub_name is None:
hub_name = uuid.UUID(int=uuid.getnode()).hex[-12:]
self.hub_name = md5(hub_name)
return self.hub_name
def is_windows(): def is_windows():
return get_platform().lower().startswith("windows") return get_platform().lower().startswith("windows")
......
...@@ -143,7 +143,7 @@ class Module(object): ...@@ -143,7 +143,7 @@ class Module(object):
if not module: if not module:
module = object.__new__(cls) module = object.__new__(cls)
else: else:
CacheUpdater(module.name, module.version).start() CacheUpdater("update_cache", module.name, module.version).start()
return module return module
def __init__(self, name=None, directory=None, module_dir=None, def __init__(self, name=None, directory=None, module_dir=None,
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册