未验证 提交 136c80a6 编写于 作者: B Bin Long 提交者: GitHub

Merge pull request #319 from ShenYuhan/add_stats_bases

Update cache.file if need
...@@ -25,6 +25,7 @@ from paddlehub.autofinetune.autoft import PSHE2 ...@@ -25,6 +25,7 @@ from paddlehub.autofinetune.autoft import PSHE2
from paddlehub.autofinetune.autoft import HAZero from paddlehub.autofinetune.autoft import HAZero
from paddlehub.autofinetune.evaluator import FullTrailEvaluator from paddlehub.autofinetune.evaluator import FullTrailEvaluator
from paddlehub.autofinetune.evaluator import PopulationBasedEvaluator from paddlehub.autofinetune.evaluator import PopulationBasedEvaluator
from paddlehub.common.hub_server import CacheUpdater
class AutoFineTuneCommand(BaseCommand): class AutoFineTuneCommand(BaseCommand):
...@@ -96,6 +97,7 @@ class AutoFineTuneCommand(BaseCommand): ...@@ -96,6 +97,7 @@ class AutoFineTuneCommand(BaseCommand):
return options_str return options_str
def execute(self, argv): def execute(self, argv):
CacheUpdater("hub_autofinetune").start()
if not argv: if not argv:
print("ERROR: Please specify a script to be finetuned in python.\n") print("ERROR: Please specify a script to be finetuned in python.\n")
self.help() self.help()
......
...@@ -21,6 +21,7 @@ import os ...@@ -21,6 +21,7 @@ import os
from paddlehub.commands.base_command import BaseCommand from paddlehub.commands.base_command import BaseCommand
from paddlehub.common.dir import CACHE_HOME from paddlehub.common.dir import CACHE_HOME
from paddlehub.common.hub_server import CacheUpdater
def file_num_in_dir(dirname): def file_num_in_dir(dirname):
...@@ -56,6 +57,7 @@ class ClearCommand(BaseCommand): ...@@ -56,6 +57,7 @@ class ClearCommand(BaseCommand):
return CACHE_HOME return CACHE_HOME
def execute(self, argv): def execute(self, argv):
CacheUpdater("hub_clear").start()
result = True result = True
total_file_size = 0 total_file_size = 0
total_file_count = 0 total_file_count = 0
......
...@@ -25,6 +25,9 @@ import re ...@@ -25,6 +25,9 @@ import re
from paddlehub.commands.base_command import BaseCommand, ENTRY from paddlehub.commands.base_command import BaseCommand, ENTRY
from paddlehub.common.dir import CONF_HOME from paddlehub.common.dir import CONF_HOME
from paddlehub.common.server_config import default_server_config from paddlehub.common.server_config import default_server_config
from paddlehub.common.hub_server import HubServer
HubServer()
class ConfigCommand(BaseCommand): class ConfigCommand(BaseCommand):
......
...@@ -24,6 +24,7 @@ import paddlehub as hub ...@@ -24,6 +24,7 @@ import paddlehub as hub
from paddlehub.common import utils from paddlehub.common import utils
from paddlehub.common.downloader import default_downloader from paddlehub.common.downloader import default_downloader
from paddlehub.commands.base_command import BaseCommand, ENTRY from paddlehub.commands.base_command import BaseCommand, ENTRY
from paddlehub.common.hub_server import CacheUpdater
class DownloadCommand(BaseCommand): class DownloadCommand(BaseCommand):
...@@ -52,6 +53,7 @@ class DownloadCommand(BaseCommand): ...@@ -52,6 +53,7 @@ class DownloadCommand(BaseCommand):
mod_name = argv[0] mod_name = argv[0]
mod_version = None if "==" not in mod_name else mod_name.split("==")[1] mod_version = None if "==" not in mod_name else mod_name.split("==")[1]
mod_name = mod_name if "==" not in mod_name else mod_name.split("==")[0] mod_name = mod_name if "==" not in mod_name else mod_name.split("==")[0]
CacheUpdater("hub_download", mod_name, mod_version).start()
self.args = self.parser.parse_args(argv[1:]) self.args = self.parser.parse_args(argv[1:])
self.args.type = self.check_type(self.args.type) self.args.type = self.check_type(self.args.type)
......
...@@ -18,6 +18,7 @@ from __future__ import division ...@@ -18,6 +18,7 @@ from __future__ import division
from __future__ import print_function from __future__ import print_function
from paddlehub.commands.base_command import BaseCommand from paddlehub.commands.base_command import BaseCommand
from paddlehub.common.hub_server import CacheUpdater
class HelpCommand(BaseCommand): class HelpCommand(BaseCommand):
...@@ -32,6 +33,7 @@ class HelpCommand(BaseCommand): ...@@ -32,6 +33,7 @@ class HelpCommand(BaseCommand):
return BaseCommand.command_dict return BaseCommand.command_dict
def execute(self, argv): def execute(self, argv):
CacheUpdater("hub_help").start()
hub_command = BaseCommand.command_dict["hub"] hub_command = BaseCommand.command_dict["hub"]
help_text = "\n" help_text = "\n"
help_text += "Usage:\n" help_text += "Usage:\n"
......
...@@ -23,6 +23,7 @@ import os ...@@ -23,6 +23,7 @@ import os
from paddlehub.common import utils from paddlehub.common import utils
from paddlehub.module.manager import default_module_manager from paddlehub.module.manager import default_module_manager
from paddlehub.commands.base_command import BaseCommand, ENTRY from paddlehub.commands.base_command import BaseCommand, ENTRY
from paddlehub.common.hub_server import CacheUpdater
class InstallCommand(BaseCommand): class InstallCommand(BaseCommand):
...@@ -57,6 +58,7 @@ class InstallCommand(BaseCommand): ...@@ -57,6 +58,7 @@ class InstallCommand(BaseCommand):
"==")[1] "==")[1]
module_name = module_name if "==" not in module_name else module_name.split( module_name = module_name if "==" not in module_name else module_name.split(
"==")[0] "==")[0]
CacheUpdater("hub_install", module_name, module_version).start()
result, tips, module_dir = default_module_manager.install_module( result, tips, module_dir = default_module_manager.install_module(
module_name=module_name, module_name=module_name,
module_version=module_version, module_version=module_version,
......
...@@ -22,6 +22,7 @@ from paddlehub.common.downloader import default_downloader ...@@ -22,6 +22,7 @@ from paddlehub.common.downloader import default_downloader
from paddlehub.module.manager import default_module_manager from paddlehub.module.manager import default_module_manager
from paddlehub.commands.base_command import BaseCommand from paddlehub.commands.base_command import BaseCommand
from paddlehub.common.cml_utils import TablePrinter from paddlehub.common.cml_utils import TablePrinter
from paddlehub.common.hub_server import CacheUpdater
class ListCommand(BaseCommand): class ListCommand(BaseCommand):
...@@ -33,6 +34,7 @@ class ListCommand(BaseCommand): ...@@ -33,6 +34,7 @@ class ListCommand(BaseCommand):
self.description = "List all installed PaddleHub modules." self.description = "List all installed PaddleHub modules."
def execute(self, argv): def execute(self, argv):
CacheUpdater("hub_list").start()
all_modules = default_module_manager.all_modules() all_modules = default_module_manager.all_modules()
if utils.is_windows(): if utils.is_windows():
placeholders = [20, 40] placeholders = [20, 40]
......
...@@ -30,6 +30,7 @@ import cv2 ...@@ -30,6 +30,7 @@ import cv2
from paddlehub.commands.base_command import BaseCommand, ENTRY from paddlehub.commands.base_command import BaseCommand, ENTRY
from paddlehub.io.parser import yaml_parser, txt_parser from paddlehub.io.parser import yaml_parser, txt_parser
from paddlehub.module.manager import default_module_manager from paddlehub.module.manager import default_module_manager
from paddlehub.common.hub_server import CacheUpdater
import paddlehub as hub import paddlehub as hub
...@@ -212,13 +213,14 @@ class RunCommand(BaseCommand): ...@@ -212,13 +213,14 @@ class RunCommand(BaseCommand):
raise RuntimeError("ERROR: Format of %s is illegal." % file_path) raise RuntimeError("ERROR: Format of %s is illegal." % file_path)
def execute(self, argv): def execute(self, argv):
if not argv: if not argv:
print("ERROR: Please specify a module name.\n") print("ERROR: Please specify a module name.\n")
self.help() self.help()
return False return False
module_name = argv[0] module_name = argv[0]
CacheUpdater("hub_run", module_name).start()
self.parser.prog = '%s %s %s' % (ENTRY, self.name, module_name) self.parser.prog = '%s %s %s' % (ENTRY, self.name, module_name)
self.arg_input_group = self.parser.add_argument_group( self.arg_input_group = self.parser.add_argument_group(
title="Input options", description="Data input to the module") title="Input options", description="Data input to the module")
......
...@@ -23,6 +23,7 @@ import paddlehub as hub ...@@ -23,6 +23,7 @@ import paddlehub as hub
from paddlehub.common import utils from paddlehub.common import utils
from paddlehub.commands.base_command import BaseCommand, ENTRY from paddlehub.commands.base_command import BaseCommand, ENTRY
from paddlehub.common.cml_utils import TablePrinter from paddlehub.common.cml_utils import TablePrinter
from paddlehub.common.hub_server import CacheUpdater
class SearchCommand(BaseCommand): class SearchCommand(BaseCommand):
...@@ -43,6 +44,7 @@ class SearchCommand(BaseCommand): ...@@ -43,6 +44,7 @@ class SearchCommand(BaseCommand):
argv = ['.*'] argv = ['.*']
resource_name = argv[0] resource_name = argv[0]
CacheUpdater("hub_search", resource_name).start()
extra = {"command": "search"} extra = {"command": "search"}
resource_list = hub.HubServer().search_resource( resource_list = hub.HubServer().search_resource(
resource_name, resource_type="Module", extra=extra) resource_name, resource_type="Module", extra=extra)
......
...@@ -25,7 +25,11 @@ import json ...@@ -25,7 +25,11 @@ import json
import paddlehub as hub import paddlehub as hub
from paddlehub.commands.base_command import BaseCommand, ENTRY from paddlehub.commands.base_command import BaseCommand, ENTRY
from paddlehub.serving import app_single as app from paddlehub.serving import app_single as app
from paddlehub.common.dir import CONF_HOME
from paddlehub.common.hub_server import CacheUpdater
import multiprocessing import multiprocessing
import time
import signal
if platform.system() == "Windows": if platform.system() == "Windows":
...@@ -64,6 +68,15 @@ def number_of_workers(): ...@@ -64,6 +68,15 @@ def number_of_workers():
return (multiprocessing.cpu_count() * 2) + 1 return (multiprocessing.cpu_count() * 2) + 1
def pid_is_exist(pid):
try:
os.kill(pid, 0)
except:
return False
else:
return True
class ServingCommand(BaseCommand): class ServingCommand(BaseCommand):
name = "serving" name = "serving"
module_list = [] module_list = []
...@@ -91,6 +104,59 @@ class ServingCommand(BaseCommand): ...@@ -91,6 +104,59 @@ class ServingCommand(BaseCommand):
self.parser.add_argument("--port", "-p", nargs="?", default=8866) self.parser.add_argument("--port", "-p", nargs="?", default=8866)
self.parser.add_argument("--gpu", "-i", nargs="?", default=0) self.parser.add_argument("--gpu", "-i", nargs="?", default=0)
def dump_pid_file(self):
pid = os.getpid()
filepath = os.path.join(CONF_HOME,
"serving_" + self.args.port + ".json")
if os.path.exists(filepath):
os.remove(filepath)
with open(filepath, "w") as fp:
info = {
"pid": pid,
"module": self.args.modules,
"start_time": time.time()
}
json.dump(info, fp)
@staticmethod
def load_pid_file(filepath, port=None):
if port is None:
port = os.path.basename(filepath).split(".")[0].split("_")[1]
if not os.path.exists(filepath):
print(
"PaddleHub-Serving config file is not exists, "
"please confirm the port [%s] you specified is correct." % port)
return False
with open(filepath, "r") as fp:
info = json.load(fp)
return info
def stop_serving(self, port):
filepath = os.path.join(CONF_HOME, "serving_" + str(port) + ".json")
info = self.load_pid_file(filepath, port)
if info is False:
return
pid = info["pid"]
module = info["module"]
start_time = info["start_time"]
if os.path.exists(filepath):
os.remove(filepath)
if not pid_is_exist(pid):
print("PaddleHub-Serving has been stopped.")
return
print("PaddleHub-Serving will stop.")
CacheUpdater(
"hub_serving_stop",
module=module,
addition={
"period_time": time.time() - start_time
}).start()
if platform.system() == "Windows":
os.kill(pid, signal.SIGTERM)
else:
os.killpg(pid, signal.SIGTERM)
@staticmethod @staticmethod
def start_bert_serving(args): def start_bert_serving(args):
if platform.system() != "Linux": if platform.system() != "Linux":
...@@ -132,6 +198,10 @@ class ServingCommand(BaseCommand): ...@@ -132,6 +198,10 @@ class ServingCommand(BaseCommand):
configs.pop() configs.pop()
module_exist.update({module_name: module_version}) module_exist.update({module_name: module_version})
try: try:
CacheUpdater(
"hub_serving_start",
module=module_name,
version=module_version).start()
m = hub.Module(name=module_name, version=module_version) m = hub.Module(name=module_name, version=module_version)
method_name = m.desc.attr.map.data['default_signature'].s method_name = m.desc.attr.map.data['default_signature'].s
if method_name == "": if method_name == "":
...@@ -147,8 +217,29 @@ class ServingCommand(BaseCommand): ...@@ -147,8 +217,29 @@ class ServingCommand(BaseCommand):
exit(1) exit(1)
return configs return configs
@staticmethod def start_app_with_file(self, configs, workers):
def start_single_app_with_file(configs): port = configs.get("port", 8866)
if ServingCommand.is_port_occupied("127.0.0.1", port) is True:
print("Port %s is occupied, please change it." % port)
return False
modules = configs.get("modules_info")
module = [str(i["module"]) + "==" + str(i["version"]) for i in modules]
module_info = ServingCommand.preinstall_modules(module)
for index in range(len(module_info)):
modules[index].update(module_info[index])
options = {
"bind": "0.0.0.0:%s" % port,
"workers": workers,
"pid": "./pid.txt"
}
configs["modules_info"] = modules
self.dump_pid_file()
StandaloneApplication(
app.create_app(init_flag=False, configs=configs), options).run()
def start_single_app_with_file(self, configs):
use_gpu = configs.get("use_gpu", False) use_gpu = configs.get("use_gpu", False)
port = configs.get("port", 8866) port = configs.get("port", 8866)
if ServingCommand.is_port_occupied("127.0.0.1", port) is True: if ServingCommand.is_port_occupied("127.0.0.1", port) is True:
...@@ -159,6 +250,7 @@ class ServingCommand(BaseCommand): ...@@ -159,6 +250,7 @@ class ServingCommand(BaseCommand):
module_info = ServingCommand.preinstall_modules(module) module_info = ServingCommand.preinstall_modules(module)
for index in range(len(module_info)): for index in range(len(module_info)):
configs[index].update(module_info[index]) configs[index].update(module_info[index])
self.dump_pid_file()
app.run(use_gpu, configs=configs, port=port) app.run(use_gpu, configs=configs, port=port)
@staticmethod @staticmethod
...@@ -173,6 +265,30 @@ class ServingCommand(BaseCommand): ...@@ -173,6 +265,30 @@ class ServingCommand(BaseCommand):
app.create_app(init_flag=False, configs=configs), options).run() app.create_app(init_flag=False, configs=configs), options).run()
print("PaddleHub-Serving has been stopped.") print("PaddleHub-Serving has been stopped.")
def start_app_with_args(self, workers):
module = self.args.modules
if module is not None:
use_gpu = self.args.use_gpu
port = self.args.port
if ServingCommand.is_port_occupied("127.0.0.1", port) is True:
print("Port %s is occupied, please change it." % port)
return False
module_info = ServingCommand.preinstall_modules(module)
[
item.update({
"batch_size": 1,
"queue_size": 20
}) for item in module_info
]
options = {"bind": "0.0.0.0:%s" % port, "workers": workers}
configs = {"use_gpu": use_gpu, "modules_info": module_info}
self.dump_pid_file()
StandaloneApplication(
app.create_app(init_flag=False, configs=configs),
options).run()
else:
print("Lack of necessary parameters!")
def start_single_app_with_args(self): def start_single_app_with_args(self):
module = self.args.modules module = self.args.modules
if module is not None: if module is not None:
...@@ -188,6 +304,7 @@ class ServingCommand(BaseCommand): ...@@ -188,6 +304,7 @@ class ServingCommand(BaseCommand):
"queue_size": 20 "queue_size": 20
}) for item in module_info }) for item in module_info
] ]
self.dump_pid_file()
app.run(use_gpu, configs=module_info, port=port) app.run(use_gpu, configs=module_info, port=port)
else: else:
print("Lack of necessary parameters!") print("Lack of necessary parameters!")
...@@ -224,52 +341,62 @@ class ServingCommand(BaseCommand): ...@@ -224,52 +341,62 @@ class ServingCommand(BaseCommand):
with open(config_file, "r") as fp: with open(config_file, "r") as fp:
configs = json.load(fp) configs = json.load(fp)
use_multiprocess = configs.get("use_multiprocess", False) use_multiprocess = configs.get("use_multiprocess", False)
if use_multiprocess is True: if platform.system() == "Windows":
if platform.system() == "Windows": print(
print( "Warning: Windows cannot use multiprocess working "
"Warning: Windows cannot use multiprocess working " "mode, Hub-Serving will switch to single process mode"
"mode, Hub-Serving will switch to single process mode" )
)
ServingCommand.start_single_app_with_file(configs)
else:
ServingCommand.start_multi_app_with_file(configs)
else:
ServingCommand.start_single_app_with_file(configs) ServingCommand.start_single_app_with_file(configs)
else:
if use_multiprocess is True:
self.start_app_with_file(configs,
number_of_workers())
else:
self.start_app_with_file(configs, 1)
else: else:
print("config_file ", config_file, "not exists.") print("config_file ", config_file, "not exists.")
else: else:
if self.args.use_multiprocess is True: if platform.system() == "Windows":
if platform.system() == "Windows": print("Warning: Windows cannot use multiprocess working "
print( "mode, Hub-Serving will switch to single process mode")
"Warning: Windows cannot use multiprocess working "
"mode, Hub-Serving will switch to single process mode")
self.start_single_app_with_args()
else:
self.start_multi_app_with_args()
else:
self.start_single_app_with_args() self.start_single_app_with_args()
else:
if self.args.use_multiprocess is True:
self.start_app_with_args(number_of_workers())
else:
self.start_app_with_args(1)
@staticmethod @staticmethod
def show_help(): def show_help():
str = "serving <option>\n" str = "serving <option>\n"
str += "\tManage PaddleHub-Serving.\n" str += "\tManage PaddleHub-Serving.\n"
str += "sub command:\n" str += "sub command:\n"
str += "start\n" str += "1. start\n"
str += "\tStart PaddleHub-Serving if specifies this parameter.\n" str += "\tStart PaddleHub-Serving if specifies this parameter.\n"
str += "start bert_service\n" str += "2. start bert_service\n"
str += "\tStart Bert Service if specifies this parameter.\n" str += "\tStart Bert Service if specifies this parameter.\n"
str += "option:\n" str += "[start] option:\n"
str += "--modules/-m [module1==version, module2==version...]\n" str += "--modules/-m [module1==version, module2==version...]\n"
str += "\tPre-install modules via this parameter list.\n" str += "\tPre-install modules via this parameter list.\n"
str += "--port/-p XXXX\n" str += "--port/-p XXXX\n"
str += "\tUse port XXXX for serving.\n" str += "\tUse port XXXX for serving.\n"
str += "--use_gpu\n" str += "--use_gpu\n"
str += "\tUse gpu for predicting if specifies this parameter.\n" str += "\tUse gpu for predicting if specifies this parameter.\n"
str += "--gpu\n" str += "--use_multiprocess\n"
str += "\tSpecify the graphics card to use, only work for Bert as Service\n" str += "\tChoose multoprocess mode, cannot be use on Windows.\n"
str += "--config/-c file_path\n" str += "--config/-c file_path\n"
str += "\tUse configs in file to starting paddlehub serving." str += "\tUse configs in file to starting paddlehub serving. "
str += "Other parameter will be ignored if specifies this parameter.\n" str += "Other parameter will be ignored if specifies this parameter.\n"
str += "[start bert_service] option:\n"
str += "--modules/-m\n"
str += "\tPre-install modules via this parameter.\n"
str += "--port/-p XXXX\n"
str += "\tUse port XXXX for serving.\n"
str += "--use_gpu\n"
str += "\tUse gpu for predicting if specifies this parameter.\n"
str += "--gpu\n"
str += "\tSpecify the graphics card to use.\n"
print(str) print(str)
def execute(self, argv): def execute(self, argv):
...@@ -285,6 +412,11 @@ class ServingCommand(BaseCommand): ...@@ -285,6 +412,11 @@ class ServingCommand(BaseCommand):
self.start_serving() self.start_serving()
else: else:
ServingCommand.show_help() ServingCommand.show_help()
elif self.args.sub_command == "stop":
if self.args.bert_service == "bert_service":
print("Please stop bert_service by kill process by yourself")
elif self.args.bert_service is None:
self.stop_serving(port=self.args.port)
else: else:
ServingCommand.show_help() ServingCommand.show_help()
......
...@@ -26,6 +26,7 @@ from paddlehub.common.cml_utils import TablePrinter ...@@ -26,6 +26,7 @@ from paddlehub.common.cml_utils import TablePrinter
from paddlehub.module.manager import default_module_manager from paddlehub.module.manager import default_module_manager
from paddlehub.module.module import Module from paddlehub.module.module import Module
from paddlehub.io.parser import yaml_parser from paddlehub.io.parser import yaml_parser
from paddlehub.common.hub_server import CacheUpdater
class ShowCommand(BaseCommand): class ShowCommand(BaseCommand):
...@@ -116,6 +117,7 @@ class ShowCommand(BaseCommand): ...@@ -116,6 +117,7 @@ class ShowCommand(BaseCommand):
return False return False
module_name = argv[0] module_name = argv[0]
CacheUpdater("hub_show", module_name).start()
# nlp model # nlp model
model_info_file = os.path.join(module_name, "info.yml") model_info_file = os.path.join(module_name, "info.yml")
......
...@@ -22,6 +22,7 @@ import argparse ...@@ -22,6 +22,7 @@ import argparse
from paddlehub.common import utils from paddlehub.common import utils
from paddlehub.module.manager import default_module_manager from paddlehub.module.manager import default_module_manager
from paddlehub.commands.base_command import BaseCommand, ENTRY from paddlehub.commands.base_command import BaseCommand, ENTRY
from paddlehub.common.hub_server import CacheUpdater
class UninstallCommand(BaseCommand): class UninstallCommand(BaseCommand):
...@@ -43,6 +44,7 @@ class UninstallCommand(BaseCommand): ...@@ -43,6 +44,7 @@ class UninstallCommand(BaseCommand):
self.help() self.help()
return False return False
module_name = argv[0] module_name = argv[0]
CacheUpdater("hub_uninstall", module_name).start()
result, tips = default_module_manager.uninstall_module( result, tips = default_module_manager.uninstall_module(
module_name=module_name) module_name=module_name)
print(tips) print(tips)
......
...@@ -19,6 +19,7 @@ from __future__ import print_function ...@@ -19,6 +19,7 @@ from __future__ import print_function
from paddlehub import version from paddlehub import version
from paddlehub.commands.base_command import BaseCommand from paddlehub.commands.base_command import BaseCommand
from paddlehub.common.hub_server import CacheUpdater
class VersionCommand(BaseCommand): class VersionCommand(BaseCommand):
...@@ -30,6 +31,7 @@ class VersionCommand(BaseCommand): ...@@ -30,6 +31,7 @@ class VersionCommand(BaseCommand):
self.description = "Show PaddleHub's version." self.description = "Show PaddleHub's version."
def execute(self, argv): def execute(self, argv):
CacheUpdater("hub_version").start()
print("hub %s" % version.hub_version) print("hub %s" % version.hub_version)
return True return True
......
#coding:utf-8
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import threading
def synchronized(func):
func.__lock__ = threading.Lock()
def synced_func(*args, **kwargs):
with func.__lock__:
return func(*args, **kwargs)
return synced_func
def singleton(cls):
_instance = {}
@synchronized
def _get_instance(*args, **kwargs):
if cls not in _instance:
_instance[cls] = cls(*args, **kwargs)
return _instance[cls]
return _get_instance
...@@ -28,37 +28,17 @@ import threading ...@@ -28,37 +28,17 @@ import threading
from paddlehub.common import utils, srv_utils from paddlehub.common import utils, srv_utils
from paddlehub.common.downloader import default_downloader from paddlehub.common.downloader import default_downloader
from paddlehub.common.decorator_utils import singleton
from paddlehub.common.server_config import default_server_config from paddlehub.common.server_config import default_server_config
from paddlehub.io.parser import yaml_parser from paddlehub.io.parser import yaml_parser
from paddlehub.common.lock import lock from paddlehub.common.lock import lock
from paddlehub.common.dir import CONF_HOME, CACHE_HOME from paddlehub.common.dir import CONF_HOME, CACHE_HOME
from paddlehub.common.srv_utils import ConfigInfo
RESOURCE_LIST_FILE = "resource_list_file.yml" RESOURCE_LIST_FILE = "resource_list_file.yml"
CACHE_TIME = 60 * 10 CACHE_TIME = 60 * 10
def synchronized(func):
func.__lock__ = threading.Lock()
def synced_func(*args, **kwargs):
with func.__lock__:
return func(*args, **kwargs)
return synced_func
def singleton(cls):
_instance = {}
@synchronized
def _get_instance(*args, **kwargs):
if cls not in _instance:
_instance[cls] = cls(*args, **kwargs)
return _instance[cls]
return _get_instance
@singleton @singleton
class HubServer(object): class HubServer(object):
def __init__(self, config_file_path=None): def __init__(self, config_file_path=None):
...@@ -72,8 +52,9 @@ class HubServer(object): ...@@ -72,8 +52,9 @@ class HubServer(object):
fp.write(json.dumps(default_server_config)) fp.write(json.dumps(default_server_config))
lock.flock(fp, lock.LOCK_UN) lock.flock(fp, lock.LOCK_UN)
with open(config_file_path) as fp: with open(config_file_path, "r") as fp:
self.config = json.load(fp) self.config = json.load(fp)
fp_lock = open(config_file_path) fp_lock = open(config_file_path)
lock.flock(fp_lock, lock.LOCK_EX) lock.flock(fp_lock, lock.LOCK_EX)
...@@ -308,29 +289,44 @@ class HubServer(object): ...@@ -308,29 +289,44 @@ class HubServer(object):
class CacheUpdater(threading.Thread): class CacheUpdater(threading.Thread):
def __init__(self, module, version=None): def __init__(self,
command="update_cache",
module=None,
version=None,
addition=None):
threading.Thread.__init__(self) threading.Thread.__init__(self)
self.command = command
self.module = module self.module = module
self.version = version self.version = version
self.addition = addition
def update_resource_list_file(self, module, version=None): def update_resource_list_file(self,
command="update_cache",
module=None,
version=None,
addition=None):
payload = {'word': module} payload = {'word': module}
if version: if version:
payload['version'] = version payload['version'] = version
api_url = srv_utils.uri_path(HubServer().get_server_url(), 'search') api_url = srv_utils.uri_path(HubServer().get_server_url(), 'search')
cache_path = os.path.join(CACHE_HOME, RESOURCE_LIST_FILE) cache_path = os.path.join(CACHE_HOME, RESOURCE_LIST_FILE)
hub_name = ConfigInfo().get_hub_name()
if os.path.exists(cache_path): if os.path.exists(cache_path):
extra = { extra = {
"command": "update_cache", "command": command,
"mtime": os.stat(cache_path).st_mtime "mtime": os.stat(cache_path).st_mtime,
"hub_name": hub_name
} }
else: else:
extra = { extra = {
"command": "update_cache", "command": command,
"mtime": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) "mtime": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
"hub_name": hub_name
} }
if addition is not None:
extra.update({"addition": addition})
try: try:
r = srv_utils.hub_request(api_url, payload, extra) r = srv_utils.hub_request(api_url, payload, extra, timeout=0.1)
if r.get("update_cache", 0) == 1: if r.get("update_cache", 0) == 1:
with open(cache_path, 'w+') as fp: with open(cache_path, 'w+') as fp:
yaml.safe_dump({'resource_list': r['data']}, fp) yaml.safe_dump({'resource_list': r['data']}, fp)
...@@ -338,7 +334,8 @@ class CacheUpdater(threading.Thread): ...@@ -338,7 +334,8 @@ class CacheUpdater(threading.Thread):
pass pass
def run(self): def run(self):
self.update_resource_list_file(self.module, self.version) self.update_resource_list_file(self.command, self.module, self.version,
self.addition)
def server_check(): def server_check():
......
...@@ -11,12 +11,17 @@ ...@@ -11,12 +11,17 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import time
import uuid
from paddlehub.common.utils import md5
HUB_SERVERS = ["http://paddlepaddle.org.cn/paddlehub"] HUB_SERVERS = ["http://paddlepaddle.org.cn/paddlehub"]
hub_name = md5(str(uuid.uuid1())[-12:]) + "-" + str(int(time.time()))
default_server_config = { default_server_config = {
"server_url": HUB_SERVERS, "server_url": HUB_SERVERS,
"resource_storage_server_url": "https://bj.bcebos.com/paddlehub-data/", "resource_storage_server_url": "https://bj.bcebos.com/paddlehub-data/",
"debug": False, "debug": False,
"log_level": "DEBUG" "log_level": "DEBUG",
"hub_name": hub_name
} }
...@@ -15,8 +15,15 @@ ...@@ -15,8 +15,15 @@
import requests import requests
import paddle import paddle
import json import json
import time
import uuid
import os
from paddlehub import version from paddlehub import version
from paddlehub.common.dir import CONF_HOME
from paddlehub.common.decorator_utils import singleton
from paddlehub.common.utils import md5
from paddlehub.common.server_config import default_server_config
def uri_path(server_url, api): def uri_path(server_url, api):
...@@ -31,9 +38,30 @@ def uri_path(server_url, api): ...@@ -31,9 +38,30 @@ def uri_path(server_url, api):
return srv return srv
def hub_request(api, params, extra=None): def hub_request(api, params, extra=None, timeout=8):
params['hub_version'] = version.hub_version params['hub_version'] = version.hub_version
params['paddle_version'] = paddle.__version__ params['paddle_version'] = paddle.__version__
params["extra"] = json.dumps(extra) params["extra"] = json.dumps(extra)
r = requests.get(api, params) r = requests.get(api, params, timeout=timeout)
return r.json() return r.json()
@singleton
class ConfigInfo(object):
def __init__(self):
self.filepath = os.path.join(CONF_HOME, "config.json")
self.hub_name = None
self.configs = None
if os.path.exists(self.filepath):
with open(self.filepath, "r") as fp:
self.configs = json.load(fp)
self.hub_name = self.configs.get("hub_name", None)
def get_hub_name(self):
if self.hub_name is None:
self.hub_name = md5(str(uuid.uuid1())[-12:]) + "-" + str(
int(time.time()))
with open(self.filepath, "w") as fp:
fp.write(json.dumps(default_server_config))
return self.hub_name
...@@ -153,7 +153,7 @@ class Module(object): ...@@ -153,7 +153,7 @@ class Module(object):
else: else:
directory = module_dir directory = module_dir
module = cls.init_with_directory(directory=directory) module = cls.init_with_directory(directory=directory)
CacheUpdater(module.name, module.version).start() CacheUpdater("update_cache", module.name, module.version).start()
else: else:
module = object.__new__(cls) module = object.__new__(cls)
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册