Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
PaddlePaddle
PaddleHub
提交
0f7b5506
P
PaddleHub
项目概览
PaddlePaddle
/
PaddleHub
大约 1 年 前同步成功
通知
282
Star
12117
Fork
2091
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
200
列表
看板
标记
里程碑
合并请求
4
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
PaddleHub
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
200
Issue
200
列表
看板
标记
里程碑
合并请求
4
合并请求
4
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
0f7b5506
编写于
1月 07, 2020
作者:
走神的阿圆
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add serving stop
上级
d3d38972
变更
19
显示空白变更内容
内联
并排
Showing
19 changed file
with
308 addition
and
79 deletion
+308
-79
paddlehub/commands/autofinetune.py
paddlehub/commands/autofinetune.py
+2
-0
paddlehub/commands/clear.py
paddlehub/commands/clear.py
+2
-0
paddlehub/commands/config.py
paddlehub/commands/config.py
+5
-0
paddlehub/commands/download.py
paddlehub/commands/download.py
+2
-0
paddlehub/commands/help.py
paddlehub/commands/help.py
+2
-0
paddlehub/commands/install.py
paddlehub/commands/install.py
+2
-0
paddlehub/commands/list.py
paddlehub/commands/list.py
+2
-0
paddlehub/commands/run.py
paddlehub/commands/run.py
+3
-1
paddlehub/commands/search.py
paddlehub/commands/search.py
+2
-0
paddlehub/commands/serving.py
paddlehub/commands/serving.py
+186
-43
paddlehub/commands/show.py
paddlehub/commands/show.py
+2
-0
paddlehub/commands/uninstall.py
paddlehub/commands/uninstall.py
+2
-0
paddlehub/commands/version.py
paddlehub/commands/version.py
+2
-0
paddlehub/common/decorator_utils.py
paddlehub/common/decorator_utils.py
+38
-0
paddlehub/common/hub_server.py
paddlehub/common/hub_server.py
+28
-31
paddlehub/common/server_config.py
paddlehub/common/server_config.py
+3
-1
paddlehub/common/srv_utils.py
paddlehub/common/srv_utils.py
+2
-2
paddlehub/common/utils.py
paddlehub/common/utils.py
+22
-0
paddlehub/module/module.py
paddlehub/module/module.py
+1
-1
未找到文件。
paddlehub/commands/autofinetune.py
浏览文件 @
0f7b5506
...
...
@@ -25,6 +25,7 @@ from paddlehub.autofinetune.autoft import PSHE2
from
paddlehub.autofinetune.autoft
import
HAZero
from
paddlehub.autofinetune.evaluator
import
FullTrailEvaluator
from
paddlehub.autofinetune.evaluator
import
PopulationBasedEvaluator
from
paddlehub.common.hub_server
import
CacheUpdater
class
AutoFineTuneCommand
(
BaseCommand
):
...
...
@@ -96,6 +97,7 @@ class AutoFineTuneCommand(BaseCommand):
return
options_str
def
execute
(
self
,
argv
):
CacheUpdater
(
"hub_autofinetune"
).
start
()
if
not
argv
:
print
(
"ERROR: Please specify a script to be finetuned in python.
\n
"
)
self
.
help
()
...
...
paddlehub/commands/clear.py
浏览文件 @
0f7b5506
...
...
@@ -21,6 +21,7 @@ import os
from
paddlehub.commands.base_command
import
BaseCommand
from
paddlehub.common.dir
import
CACHE_HOME
from
paddlehub.common.hub_server
import
CacheUpdater
def
file_num_in_dir
(
dirname
):
...
...
@@ -56,6 +57,7 @@ class ClearCommand(BaseCommand):
return
CACHE_HOME
def
execute
(
self
,
argv
):
CacheUpdater
(
"hub_clear"
).
start
()
result
=
True
total_file_size
=
0
total_file_count
=
0
...
...
paddlehub/commands/config.py
浏览文件 @
0f7b5506
...
...
@@ -25,6 +25,10 @@ import re
from
paddlehub.commands.base_command
import
BaseCommand
,
ENTRY
from
paddlehub.common.dir
import
CONF_HOME
from
paddlehub.common.server_config
import
default_server_config
from
paddlehub.common.hub_server
import
CacheUpdater
from
paddlehub.common.hub_server
import
HubServer
HubServer
()
class
ConfigCommand
(
BaseCommand
):
...
...
@@ -98,6 +102,7 @@ class ConfigCommand(BaseCommand):
print
(
str
)
def
execute
(
self
,
argv
):
CacheUpdater
(
"hub_config"
).
start
()
args
=
self
.
parser
.
parse_args
()
if
args
.
option
is
None
:
ConfigCommand
.
show_config
()
...
...
paddlehub/commands/download.py
浏览文件 @
0f7b5506
...
...
@@ -24,6 +24,7 @@ import paddlehub as hub
from
paddlehub.common
import
utils
from
paddlehub.common.downloader
import
default_downloader
from
paddlehub.commands.base_command
import
BaseCommand
,
ENTRY
from
paddlehub.common.hub_server
import
CacheUpdater
class
DownloadCommand
(
BaseCommand
):
...
...
@@ -52,6 +53,7 @@ class DownloadCommand(BaseCommand):
mod_name
=
argv
[
0
]
mod_version
=
None
if
"=="
not
in
mod_name
else
mod_name
.
split
(
"=="
)[
1
]
mod_name
=
mod_name
if
"=="
not
in
mod_name
else
mod_name
.
split
(
"=="
)[
0
]
CacheUpdater
(
"hub_download"
,
mod_name
,
mod_version
).
start
()
self
.
args
=
self
.
parser
.
parse_args
(
argv
[
1
:])
self
.
args
.
type
=
self
.
check_type
(
self
.
args
.
type
)
...
...
paddlehub/commands/help.py
浏览文件 @
0f7b5506
...
...
@@ -18,6 +18,7 @@ from __future__ import division
from
__future__
import
print_function
from
paddlehub.commands.base_command
import
BaseCommand
from
paddlehub.common.hub_server
import
CacheUpdater
class
HelpCommand
(
BaseCommand
):
...
...
@@ -32,6 +33,7 @@ class HelpCommand(BaseCommand):
return
BaseCommand
.
command_dict
def
execute
(
self
,
argv
):
CacheUpdater
(
"hub_help"
).
start
()
hub_command
=
BaseCommand
.
command_dict
[
"hub"
]
help_text
=
"
\n
"
help_text
+=
"Usage:
\n
"
...
...
paddlehub/commands/install.py
浏览文件 @
0f7b5506
...
...
@@ -23,6 +23,7 @@ import os
from
paddlehub.common
import
utils
from
paddlehub.module.manager
import
default_module_manager
from
paddlehub.commands.base_command
import
BaseCommand
,
ENTRY
from
paddlehub.common.hub_server
import
CacheUpdater
class
InstallCommand
(
BaseCommand
):
...
...
@@ -57,6 +58,7 @@ class InstallCommand(BaseCommand):
"=="
)[
1
]
module_name
=
module_name
if
"=="
not
in
module_name
else
module_name
.
split
(
"=="
)[
0
]
CacheUpdater
(
"hub_install"
,
module_name
,
module_version
).
start
()
result
,
tips
,
module_dir
=
default_module_manager
.
install_module
(
module_name
=
module_name
,
module_version
=
module_version
,
...
...
paddlehub/commands/list.py
浏览文件 @
0f7b5506
...
...
@@ -22,6 +22,7 @@ from paddlehub.common.downloader import default_downloader
from
paddlehub.module.manager
import
default_module_manager
from
paddlehub.commands.base_command
import
BaseCommand
from
paddlehub.common.cml_utils
import
TablePrinter
from
paddlehub.common.hub_server
import
CacheUpdater
class
ListCommand
(
BaseCommand
):
...
...
@@ -33,6 +34,7 @@ class ListCommand(BaseCommand):
self
.
description
=
"List all installed PaddleHub modules."
def
execute
(
self
,
argv
):
CacheUpdater
(
"hub_list"
).
start
()
all_modules
=
default_module_manager
.
all_modules
()
if
utils
.
is_windows
():
placeholders
=
[
20
,
40
]
...
...
paddlehub/commands/run.py
浏览文件 @
0f7b5506
...
...
@@ -30,6 +30,7 @@ import cv2
from
paddlehub.commands.base_command
import
BaseCommand
,
ENTRY
from
paddlehub.io.parser
import
yaml_parser
,
txt_parser
from
paddlehub.module.manager
import
default_module_manager
from
paddlehub.common.hub_server
import
CacheUpdater
import
paddlehub
as
hub
...
...
@@ -212,13 +213,14 @@ class RunCommand(BaseCommand):
raise
RuntimeError
(
"ERROR: Format of %s is illegal."
%
file_path
)
def
execute
(
self
,
argv
):
if
not
argv
:
print
(
"ERROR: Please specify a module name.
\n
"
)
self
.
help
()
return
False
module_name
=
argv
[
0
]
CacheUpdater
(
"hub_run"
,
module_name
).
start
()
self
.
parser
.
prog
=
'%s %s %s'
%
(
ENTRY
,
self
.
name
,
module_name
)
self
.
arg_input_group
=
self
.
parser
.
add_argument_group
(
title
=
"Input options"
,
description
=
"Data input to the module"
)
...
...
paddlehub/commands/search.py
浏览文件 @
0f7b5506
...
...
@@ -23,6 +23,7 @@ import paddlehub as hub
from
paddlehub.common
import
utils
from
paddlehub.commands.base_command
import
BaseCommand
,
ENTRY
from
paddlehub.common.cml_utils
import
TablePrinter
from
paddlehub.common.hub_server
import
CacheUpdater
class
SearchCommand
(
BaseCommand
):
...
...
@@ -43,6 +44,7 @@ class SearchCommand(BaseCommand):
argv
=
[
'.*'
]
resource_name
=
argv
[
0
]
CacheUpdater
(
"hub_search"
,
resource_name
).
start
()
extra
=
{
"command"
:
"search"
}
resource_list
=
hub
.
HubServer
().
search_resource
(
resource_name
,
resource_type
=
"Module"
,
extra
=
extra
)
...
...
paddlehub/commands/serving.py
浏览文件 @
0f7b5506
...
...
@@ -25,15 +25,27 @@ import json
import
paddlehub
as
hub
from
paddlehub.commands.base_command
import
BaseCommand
,
ENTRY
from
paddlehub.serving
import
app_single
as
app
from
paddlehub.common.dir
import
CONF_HOME
from
paddlehub.common.hub_server
import
CacheUpdater
import
multiprocessing
import
gunicorn.app.base
import
time
import
signal
if
platform
.
system
()
==
"Windows"
:
def
number_of_workers
():
return
(
multiprocessing
.
cpu_count
()
*
2
)
+
1
class
StandaloneApplication
(
object
):
def
__init__
(
self
):
pass
def
load_config
(
self
):
pass
def
load
(
self
):
pass
else
:
import
gunicorn.app.base
class
StandaloneApplication
(
gunicorn
.
app
.
base
.
BaseApplication
):
class
StandaloneApplication
(
gunicorn
.
app
.
base
.
BaseApplication
):
def
__init__
(
self
,
app
,
options
=
None
):
self
.
options
=
options
or
{}
self
.
application
=
app
...
...
@@ -52,6 +64,19 @@ class StandaloneApplication(gunicorn.app.base.BaseApplication):
return
self
.
application
def
number_of_workers
():
return
(
multiprocessing
.
cpu_count
()
*
2
)
+
1
def
pid_is_exist
(
pid
):
try
:
os
.
kill
(
pid
,
0
)
except
:
return
False
else
:
return
True
class
ServingCommand
(
BaseCommand
):
name
=
"serving"
module_list
=
[]
...
...
@@ -79,6 +104,59 @@ class ServingCommand(BaseCommand):
self
.
parser
.
add_argument
(
"--port"
,
"-p"
,
nargs
=
"?"
,
default
=
8866
)
self
.
parser
.
add_argument
(
"--gpu"
,
"-i"
,
nargs
=
"?"
,
default
=
0
)
def
dump_pid_file
(
self
):
pid
=
os
.
getpid
()
filepath
=
os
.
path
.
join
(
CONF_HOME
,
"serving_"
+
self
.
args
.
port
+
".json"
)
if
os
.
path
.
exists
(
filepath
):
os
.
remove
(
filepath
)
with
open
(
filepath
,
"w"
)
as
fp
:
info
=
{
"pid"
:
pid
,
"module"
:
self
.
args
.
modules
,
"start_time"
:
time
.
time
()
}
json
.
dump
(
info
,
fp
)
@
staticmethod
def
load_pid_file
(
filepath
,
port
=
None
):
if
port
is
None
:
port
=
os
.
path
.
basename
(
filepath
).
split
(
"."
)[
0
].
split
(
"_"
)[
1
]
if
not
os
.
path
.
exists
(
filepath
):
print
(
"PaddleHub-Serving config file is not exists, "
"please confirm the port [%s] you specified is correct."
%
port
)
return
False
with
open
(
filepath
,
"r"
)
as
fp
:
info
=
json
.
load
(
fp
)
return
info
def
stop_serving
(
self
,
port
):
filepath
=
os
.
path
.
join
(
CONF_HOME
,
"serving_"
+
str
(
port
)
+
".json"
)
info
=
self
.
load_pid_file
(
filepath
,
port
)
if
info
is
False
:
return
pid
=
info
[
"pid"
]
module
=
info
[
"module"
]
start_time
=
info
[
"start_time"
]
if
os
.
path
.
exists
(
filepath
):
os
.
remove
(
filepath
)
if
not
pid_is_exist
(
pid
):
print
(
"PaddleHub-Serving has been stopped."
)
return
print
(
"PaddleHub-Serving will stop."
)
CacheUpdater
(
"hub_serving_stop"
,
module
=
module
,
addition
=
{
"period_time"
:
time
.
time
()
-
start_time
}).
start
()
if
platform
.
system
()
==
"Windows"
:
os
.
kill
(
pid
,
signal
.
SIGTERM
)
else
:
os
.
killpg
(
pid
,
signal
.
SIGTERM
)
@
staticmethod
def
start_bert_serving
(
args
):
if
platform
.
system
()
!=
"Linux"
:
...
...
@@ -120,6 +198,10 @@ class ServingCommand(BaseCommand):
configs
.
pop
()
module_exist
.
update
({
module_name
:
module_version
})
try
:
CacheUpdater
(
"hub_serving_start"
,
module
=
module_name
,
version
=
module_version
).
start
()
m
=
hub
.
Module
(
name
=
module_name
,
version
=
module_version
)
method_name
=
m
.
desc
.
attr
.
map
.
data
[
'default_signature'
].
s
if
method_name
==
""
:
...
...
@@ -135,6 +217,28 @@ class ServingCommand(BaseCommand):
exit
(
1
)
return
configs
def
start_app_with_file
(
self
,
configs
,
workers
):
port
=
configs
.
get
(
"port"
,
8866
)
if
ServingCommand
.
is_port_occupied
(
"127.0.0.1"
,
port
)
is
True
:
print
(
"Port %s is occupied, please change it."
%
port
)
return
False
modules
=
configs
.
get
(
"modules_info"
)
module
=
[
str
(
i
[
"module"
])
+
"=="
+
str
(
i
[
"version"
])
for
i
in
modules
]
module_info
=
ServingCommand
.
preinstall_modules
(
module
)
for
index
in
range
(
len
(
module_info
)):
modules
[
index
].
update
(
module_info
[
index
])
options
=
{
"bind"
:
"0.0.0.0:%s"
%
port
,
"workers"
:
workers
,
"pid"
:
"./pid.txt"
}
configs
[
"modules_info"
]
=
modules
self
.
dump_pid_file
()
StandaloneApplication
(
app
.
create_app
(
init_flag
=
False
,
configs
=
configs
),
options
).
run
()
@
staticmethod
def
start_single_app_with_file
(
configs
):
use_gpu
=
configs
.
get
(
"use_gpu"
,
False
)
...
...
@@ -161,6 +265,30 @@ class ServingCommand(BaseCommand):
app
.
create_app
(
init_flag
=
False
,
configs
=
configs
),
options
).
run
()
print
(
"PaddleHub-Serving has been stopped."
)
def
start_app_with_args
(
self
,
workers
):
module
=
self
.
args
.
modules
if
module
is
not
None
:
use_gpu
=
self
.
args
.
use_gpu
port
=
self
.
args
.
port
if
ServingCommand
.
is_port_occupied
(
"127.0.0.1"
,
port
)
is
True
:
print
(
"Port %s is occupied, please change it."
%
port
)
return
False
module_info
=
ServingCommand
.
preinstall_modules
(
module
)
[
item
.
update
({
"batch_size"
:
1
,
"queue_size"
:
20
})
for
item
in
module_info
]
options
=
{
"bind"
:
"0.0.0.0:%s"
%
port
,
"workers"
:
workers
}
configs
=
{
"use_gpu"
:
use_gpu
,
"modules_info"
:
module_info
}
self
.
dump_pid_file
()
StandaloneApplication
(
app
.
create_app
(
init_flag
=
False
,
configs
=
configs
),
options
).
run
()
else
:
print
(
"Lack of necessary parameters!"
)
def
start_single_app_with_args
(
self
):
module
=
self
.
args
.
modules
if
module
is
not
None
:
...
...
@@ -212,7 +340,6 @@ class ServingCommand(BaseCommand):
with
open
(
config_file
,
"r"
)
as
fp
:
configs
=
json
.
load
(
fp
)
use_multiprocess
=
configs
.
get
(
"use_multiprocess"
,
False
)
if
use_multiprocess
is
True
:
if
platform
.
system
()
==
"Windows"
:
print
(
"Warning: Windows cannot use multiprocess working "
...
...
@@ -220,44 +347,55 @@ class ServingCommand(BaseCommand):
)
ServingCommand
.
start_single_app_with_file
(
configs
)
else
:
ServingCommand
.
start_multi_app_with_file
(
configs
)
if
use_multiprocess
is
True
:
self
.
start_app_with_file
(
configs
,
number_of_workers
())
else
:
ServingCommand
.
start_single_app_with_file
(
configs
)
self
.
start_app_with_file
(
configs
,
1
)
else
:
print
(
"config_file "
,
config_file
,
"not exists."
)
else
:
if
self
.
args
.
use_multiprocess
is
True
:
if
platform
.
system
()
==
"Windows"
:
print
(
"Warning: Windows cannot use multiprocess working "
print
(
"Warning: Windows cannot use multiprocess working "
"mode, Hub-Serving will switch to single process mode"
)
self
.
start_single_app_with_args
()
else
:
self
.
start_multi_app_with_args
()
if
self
.
args
.
use_multiprocess
is
True
:
self
.
start_app_with_args
(
number_of_workers
())
else
:
self
.
start_single_app_with_args
(
)
self
.
start_app_with_args
(
1
)
@
staticmethod
def
show_help
():
str
=
"serving <option>
\n
"
str
+=
"
\t
Manage PaddleHub-Serving.
\n
"
str
+=
"sub command:
\n
"
str
+=
"start
\n
"
str
+=
"
1.
start
\n
"
str
+=
"
\t
Start PaddleHub-Serving if specifies this parameter.
\n
"
str
+=
"start bert_service
\n
"
str
+=
"
2.
start bert_service
\n
"
str
+=
"
\t
Start Bert Service if specifies this parameter.
\n
"
str
+=
"option:
\n
"
str
+=
"
[start]
option:
\n
"
str
+=
"--modules/-m [module1==version, module2==version...]
\n
"
str
+=
"
\t
Pre-install modules via this parameter list.
\n
"
str
+=
"--port/-p XXXX
\n
"
str
+=
"
\t
Use port XXXX for serving.
\n
"
str
+=
"--use_gpu
\n
"
str
+=
"
\t
Use gpu for predicting if specifies this parameter.
\n
"
str
+=
"--
gpu
\n
"
str
+=
"
\t
Specify the graphics card to use, only work for Bert as Service
\n
"
str
+=
"--
use_multiprocess
\n
"
str
+=
"
\t
Choose multoprocess mode, cannot be use on Windows.
\n
"
str
+=
"--config/-c file_path
\n
"
str
+=
"
\t
Use configs in file to starting paddlehub serving."
str
+=
"
\t
Use configs in file to starting paddlehub serving.
"
str
+=
"Other parameter will be ignored if specifies this parameter.
\n
"
str
+=
"[start bert_service] option:
\n
"
str
+=
"--modules/-m
\n
"
str
+=
"
\t
Pre-install modules via this parameter.
\n
"
str
+=
"--port/-p XXXX
\n
"
str
+=
"
\t
Use port XXXX for serving.
\n
"
str
+=
"--use_gpu
\n
"
str
+=
"
\t
Use gpu for predicting if specifies this parameter.
\n
"
str
+=
"--gpu
\n
"
str
+=
"
\t
Specify the graphics card to use.
\n
"
print
(
str
)
def
execute
(
self
,
argv
):
...
...
@@ -273,6 +411,11 @@ class ServingCommand(BaseCommand):
self
.
start_serving
()
else
:
ServingCommand
.
show_help
()
elif
self
.
args
.
sub_command
==
"stop"
:
if
self
.
args
.
bert_service
==
"bert_service"
:
print
(
"Please stop bert_service by kill process by yourself"
)
elif
self
.
args
.
bert_service
is
None
:
self
.
stop_serving
(
port
=
self
.
args
.
port
)
else
:
ServingCommand
.
show_help
()
...
...
paddlehub/commands/show.py
浏览文件 @
0f7b5506
...
...
@@ -26,6 +26,7 @@ from paddlehub.common.cml_utils import TablePrinter
from
paddlehub.module.manager
import
default_module_manager
from
paddlehub.module.module
import
Module
from
paddlehub.io.parser
import
yaml_parser
from
paddlehub.common.hub_server
import
CacheUpdater
class
ShowCommand
(
BaseCommand
):
...
...
@@ -116,6 +117,7 @@ class ShowCommand(BaseCommand):
return
False
module_name
=
argv
[
0
]
CacheUpdater
(
"hub_show"
,
module_name
).
start
()
# nlp model
model_info_file
=
os
.
path
.
join
(
module_name
,
"info.yml"
)
...
...
paddlehub/commands/uninstall.py
浏览文件 @
0f7b5506
...
...
@@ -22,6 +22,7 @@ import argparse
from
paddlehub.common
import
utils
from
paddlehub.module.manager
import
default_module_manager
from
paddlehub.commands.base_command
import
BaseCommand
,
ENTRY
from
paddlehub.common.hub_server
import
CacheUpdater
class
UninstallCommand
(
BaseCommand
):
...
...
@@ -43,6 +44,7 @@ class UninstallCommand(BaseCommand):
self
.
help
()
return
False
module_name
=
argv
[
0
]
CacheUpdater
(
"hub_uninstall"
,
module_name
).
start
()
result
,
tips
=
default_module_manager
.
uninstall_module
(
module_name
=
module_name
)
print
(
tips
)
...
...
paddlehub/commands/version.py
浏览文件 @
0f7b5506
...
...
@@ -19,6 +19,7 @@ from __future__ import print_function
from
paddlehub
import
version
from
paddlehub.commands.base_command
import
BaseCommand
from
paddlehub.common.hub_server
import
CacheUpdater
class
VersionCommand
(
BaseCommand
):
...
...
@@ -30,6 +31,7 @@ class VersionCommand(BaseCommand):
self
.
description
=
"Show PaddleHub's version."
def
execute
(
self
,
argv
):
CacheUpdater
(
"hub_version"
).
start
()
print
(
"hub %s"
%
version
.
hub_version
)
return
True
...
...
paddlehub/common/decorator_utils.py
0 → 100644
浏览文件 @
0f7b5506
#coding:utf-8
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import
threading
def
synchronized
(
func
):
func
.
__lock__
=
threading
.
Lock
()
def
synced_func
(
*
args
,
**
kwargs
):
with
func
.
__lock__
:
return
func
(
*
args
,
**
kwargs
)
return
synced_func
def
singleton
(
cls
):
_instance
=
{}
@
synchronized
def
_get_instance
(
*
args
,
**
kwargs
):
if
cls
not
in
_instance
:
_instance
[
cls
]
=
cls
(
*
args
,
**
kwargs
)
return
_instance
[
cls
]
return
_get_instance
paddlehub/common/hub_server.py
浏览文件 @
0f7b5506
...
...
@@ -28,37 +28,17 @@ import threading
from
paddlehub.common
import
utils
,
srv_utils
from
paddlehub.common.downloader
import
default_downloader
from
paddlehub.common.decorator_utils
import
singleton
from
paddlehub.common.server_config
import
default_server_config
from
paddlehub.io.parser
import
yaml_parser
from
paddlehub.common.lock
import
lock
from
paddlehub.common.dir
import
CONF_HOME
,
CACHE_HOME
from
paddlehub.common.utils
import
ConfigInfo
RESOURCE_LIST_FILE
=
"resource_list_file.yml"
CACHE_TIME
=
60
*
10
def
synchronized
(
func
):
func
.
__lock__
=
threading
.
Lock
()
def
synced_func
(
*
args
,
**
kwargs
):
with
func
.
__lock__
:
return
func
(
*
args
,
**
kwargs
)
return
synced_func
def
singleton
(
cls
):
_instance
=
{}
@
synchronized
def
_get_instance
(
*
args
,
**
kwargs
):
if
cls
not
in
_instance
:
_instance
[
cls
]
=
cls
(
*
args
,
**
kwargs
)
return
_instance
[
cls
]
return
_get_instance
@
singleton
class
HubServer
(
object
):
def
__init__
(
self
,
config_file_path
=
None
):
...
...
@@ -72,8 +52,9 @@ class HubServer(object):
fp
.
write
(
json
.
dumps
(
default_server_config
))
lock
.
flock
(
fp
,
lock
.
LOCK_UN
)
with
open
(
config_file_path
)
as
fp
:
with
open
(
config_file_path
,
"r"
)
as
fp
:
self
.
config
=
json
.
load
(
fp
)
fp_lock
=
open
(
config_file_path
)
lock
.
flock
(
fp_lock
,
lock
.
LOCK_EX
)
...
...
@@ -308,29 +289,44 @@ class HubServer(object):
class
CacheUpdater
(
threading
.
Thread
):
def
__init__
(
self
,
module
,
version
=
None
):
def
__init__
(
self
,
command
=
"update_cache"
,
module
=
None
,
version
=
None
,
addition
=
None
):
threading
.
Thread
.
__init__
(
self
)
self
.
command
=
command
self
.
module
=
module
self
.
version
=
version
self
.
addition
=
addition
def
update_resource_list_file
(
self
,
module
,
version
=
None
):
def
update_resource_list_file
(
self
,
command
=
"update_cache"
,
module
=
None
,
version
=
None
,
addition
=
None
):
payload
=
{
'word'
:
module
}
if
version
:
payload
[
'version'
]
=
version
api_url
=
srv_utils
.
uri_path
(
HubServer
().
get_server_url
(),
'search'
)
cache_path
=
os
.
path
.
join
(
CACHE_HOME
,
RESOURCE_LIST_FILE
)
hub_name
=
ConfigInfo
().
get_hub_name
()
if
os
.
path
.
exists
(
cache_path
):
extra
=
{
"command"
:
"update_cache"
,
"mtime"
:
os
.
stat
(
cache_path
).
st_mtime
"command"
:
command
,
"mtime"
:
os
.
stat
(
cache_path
).
st_mtime
,
"hub_name"
:
hub_name
}
else
:
extra
=
{
"command"
:
"update_cache"
,
"mtime"
:
time
.
strftime
(
"%Y-%m-%d %H:%M:%S"
,
time
.
localtime
())
"command"
:
command
,
"mtime"
:
time
.
strftime
(
"%Y-%m-%d %H:%M:%S"
,
time
.
localtime
()),
"hub_name"
:
hub_name
}
if
addition
is
not
None
:
extra
.
update
({
"addition"
:
addition
})
try
:
r
=
srv_utils
.
hub_request
(
api_url
,
payload
,
extra
)
r
=
srv_utils
.
hub_request
(
api_url
,
payload
,
extra
,
timeout
=
0.1
)
if
r
.
get
(
"update_cache"
,
0
)
==
1
:
with
open
(
cache_path
,
'w+'
)
as
fp
:
yaml
.
safe_dump
({
'resource_list'
:
r
[
'data'
]},
fp
)
...
...
@@ -338,7 +334,8 @@ class CacheUpdater(threading.Thread):
pass
def
run
(
self
):
self
.
update_resource_list_file
(
self
.
module
,
self
.
version
)
self
.
update_resource_list_file
(
self
.
command
,
self
.
module
,
self
.
version
,
self
.
addition
)
def
server_check
():
...
...
paddlehub/common/server_config.py
浏览文件 @
0f7b5506
...
...
@@ -11,6 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from
paddlehub.common.utils
import
ConfigInfo
HUB_SERVERS
=
[
"http://paddlepaddle.org.cn/paddlehub"
]
...
...
@@ -18,5 +19,6 @@ default_server_config = {
"server_url"
:
HUB_SERVERS
,
"resource_storage_server_url"
:
"https://bj.bcebos.com/paddlehub-data/"
,
"debug"
:
False
,
"log_level"
:
"DEBUG"
"log_level"
:
"DEBUG"
,
"hub_name"
:
ConfigInfo
().
get_hub_name
()
}
paddlehub/common/srv_utils.py
浏览文件 @
0f7b5506
...
...
@@ -31,9 +31,9 @@ def uri_path(server_url, api):
return
srv
def
hub_request
(
api
,
params
,
extra
=
None
):
def
hub_request
(
api
,
params
,
extra
=
None
,
timeout
=
8
):
params
[
'hub_version'
]
=
version
.
hub_version
params
[
'paddle_version'
]
=
paddle
.
__version__
params
[
"extra"
]
=
json
.
dumps
(
extra
)
r
=
requests
.
get
(
api
,
params
)
r
=
requests
.
get
(
api
,
params
,
timeout
=
timeout
)
return
r
.
json
()
paddlehub/common/utils.py
浏览文件 @
0f7b5506
...
...
@@ -22,12 +22,16 @@ import os
import
multiprocessing
import
hashlib
import
platform
import
uuid
import
json
import
paddle.fluid
as
fluid
import
six
from
paddlehub.module
import
module_desc_pb2
from
paddlehub.common.logger
import
logger
from
paddlehub.common.dir
import
CONF_HOME
from
paddlehub.common.decorator_utils
import
singleton
def
version_compare
(
version1
,
version2
):
...
...
@@ -55,6 +59,24 @@ def get_platform():
return
platform
.
platform
()
@
singleton
class
ConfigInfo
(
object
):
def
__init__
(
self
):
self
.
filepath
=
os
.
path
.
join
(
CONF_HOME
,
"config.json"
)
self
.
hub_name
=
None
self
.
configs
=
None
if
os
.
path
.
exists
(
self
.
filepath
):
with
open
(
self
.
filepath
,
"r"
)
as
fp
:
self
.
configs
=
json
.
load
(
fp
)
self
.
use_id
=
self
.
configs
.
get
(
"hub_name"
,
None
)
def
get_hub_name
(
self
):
if
self
.
hub_name
is
None
:
hub_name
=
uuid
.
UUID
(
int
=
uuid
.
getnode
()).
hex
[
-
12
:]
self
.
hub_name
=
md5
(
hub_name
)
return
self
.
hub_name
def
is_windows
():
return
get_platform
().
lower
().
startswith
(
"windows"
)
...
...
paddlehub/module/module.py
浏览文件 @
0f7b5506
...
...
@@ -143,7 +143,7 @@ class Module(object):
if
not
module
:
module
=
object
.
__new__
(
cls
)
else
:
CacheUpdater
(
module
.
name
,
module
.
version
).
start
()
CacheUpdater
(
"update_cache"
,
module
.
name
,
module
.
version
).
start
()
return
module
def
__init__
(
self
,
name
=
None
,
directory
=
None
,
module_dir
=
None
,
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录