Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
PaddlePaddle
PaddleHub
提交
eb0785c1
P
PaddleHub
项目概览
PaddlePaddle
/
PaddleHub
大约 1 年 前同步成功
通知
282
Star
12117
Fork
2091
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
200
列表
看板
标记
里程碑
合并请求
4
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
PaddleHub
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
200
Issue
200
列表
看板
标记
里程碑
合并请求
4
合并请求
4
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
未验证
提交
eb0785c1
编写于
3月 01, 2020
作者:
走神的阿圆
提交者:
GitHub
3月 01, 2020
浏览文件
操作
浏览文件
下载
差异文件
Merge branch 'develop' into update_v2_0301
上级
c223d381
16f7e36f
变更
5
显示空白变更内容
内联
并排
Showing
5 changed file
with
118 addition
and
147 deletion
+118
-147
paddlehub/commands/install.py
paddlehub/commands/install.py
+1
-1
paddlehub/module/check_info.proto
paddlehub/module/check_info.proto
+0
-1
paddlehub/module/check_info_pb2.py
paddlehub/module/check_info_pb2.py
+6
-22
paddlehub/module/manager.py
paddlehub/module/manager.py
+19
-0
paddlehub/module/module.py
paddlehub/module/module.py
+92
-123
未找到文件。
paddlehub/commands/install.py
浏览文件 @
eb0785c1
...
...
@@ -46,7 +46,7 @@ class InstallCommand(BaseCommand):
return
False
extra
=
{
"command"
:
"install"
}
if
argv
[
0
].
endswith
(
"tar.gz"
)
or
argv
[
0
].
endswith
(
"phm"
)
:
if
argv
[
0
].
endswith
(
"tar.gz"
):
result
,
tips
,
module_dir
=
default_module_manager
.
install_module
(
module_package
=
argv
[
0
],
extra
=
extra
)
elif
os
.
path
.
exists
(
argv
[
0
])
and
os
.
path
.
isdir
(
argv
[
0
]):
...
...
paddlehub/module/check_info.proto
浏览文件 @
eb0785c1
...
...
@@ -52,5 +52,4 @@ message CheckInfo {
string
module_proto_version
=
3
;
repeated
FileInfo
file_infos
=
4
;
repeated
Requires
requires
=
5
;
string
module_code_version
=
6
;
};
paddlehub/module/check_info_pb2.py
浏览文件 @
eb0785c1
...
...
@@ -18,7 +18,7 @@ DESCRIPTOR = _descriptor.FileDescriptor(
package
=
'paddlehub.module.checkinfo'
,
syntax
=
'proto3'
,
serialized_pb
=
_b
(
'
\n\x10\x63
heck_info.proto
\x12\x1a
paddlehub.module.checkinfo
\"\x85\x01\n\x08\x46
ileInfo
\x12\x11\n\t
file_name
\x18\x01
\x01
(
\t\x12\x33\n\x04
type
\x18\x02
\x01
(
\x0e\x32
%.paddlehub.module.checkinfo.FILE_TYPE
\x12\x0f\n\x07
is_need
\x18\x03
\x01
(
\x08\x12\x0b\n\x03
md5
\x18\x04
\x01
(
\t\x12\x13\n\x0b\x64\x65
scription
\x18\x05
\x01
(
\t\"\x84\x01\n\x08
Requires
\x12
>
\n\x0c
require_type
\x18\x01
\x01
(
\x0e\x32
(.paddlehub.module.checkinfo.REQUIRE_TYPE
\x12\x0f\n\x07
version
\x18\x02
\x01
(
\t\x12\x12\n\n
great_than
\x18\x03
\x01
(
\x08\x12\x13\n\x0b\x64\x65
scription
\x18\x04
\x01
(
\t\"\x
e5\x01\n\t
CheckInfo
\x12\x16\n\x0e
paddle_version
\x18\x01
\x01
(
\t\x12\x13\n\x0b
hub_version
\x18\x02
\x01
(
\t\x12\x1c\n\x14
module_proto_version
\x18\x03
\x01
(
\t\x12\x38\n\n
file_infos
\x18\x04
\x03
(
\x0b\x32
$.paddlehub.module.checkinfo.FileInfo
\x12\x36\n\x08
requires
\x18\x05
\x03
(
\x0b\x32
$.paddlehub.module.checkinfo.Requires
\x12\x1b\n\x13
module_code_version
\x18\x06
\x01
(
\t
*
\x1e\n\t
FILE_TYPE
\x12\x08\n\x04\x46
ILE
\x10\x00\x12\x07\n\x03\x44
IR
\x10\x01
*[
\n\x0c
REQUIRE_TYPE
\x12\x12\n\x0e
PYTHON_PACKAGE
\x10\x00\x12\x0e\n\n
HUB_MODULE
\x10\x01\x12\n\n\x06
SYSTEM
\x10\x02\x12\x0b\n\x07\x43
OMMAND
\x10\x03\x12\x0e\n\n
PY_VERSION
\x10\x04\x42\x02
H
\x03\x62\x06
proto3'
'
\n\x10\x63
heck_info.proto
\x12\x1a
paddlehub.module.checkinfo
\"\x85\x01\n\x08\x46
ileInfo
\x12\x11\n\t
file_name
\x18\x01
\x01
(
\t\x12\x33\n\x04
type
\x18\x02
\x01
(
\x0e\x32
%.paddlehub.module.checkinfo.FILE_TYPE
\x12\x0f\n\x07
is_need
\x18\x03
\x01
(
\x08\x12\x0b\n\x03
md5
\x18\x04
\x01
(
\t\x12\x13\n\x0b\x64\x65
scription
\x18\x05
\x01
(
\t\"\x84\x01\n\x08
Requires
\x12
>
\n\x0c
require_type
\x18\x01
\x01
(
\x0e\x32
(.paddlehub.module.checkinfo.REQUIRE_TYPE
\x12\x0f\n\x07
version
\x18\x02
\x01
(
\t\x12\x12\n\n
great_than
\x18\x03
\x01
(
\x08\x12\x13\n\x0b\x64\x65
scription
\x18\x04
\x01
(
\t\"\x
c8\x01\n\t
CheckInfo
\x12\x16\n\x0e
paddle_version
\x18\x01
\x01
(
\t\x12\x13\n\x0b
hub_version
\x18\x02
\x01
(
\t\x12\x1c\n\x14
module_proto_version
\x18\x03
\x01
(
\t\x12\x38\n\n
file_infos
\x18\x04
\x03
(
\x0b\x32
$.paddlehub.module.checkinfo.FileInfo
\x12\x36\n\x08
requires
\x18\x05
\x03
(
\x0b\x32
$.paddlehub.module.checkinfo.Requires
*
\x1e\n\t
FILE_TYPE
\x12\x08\n\x04\x46
ILE
\x10\x00\x12\x07\n\x03\x44
IR
\x10\x01
*[
\n\x0c
REQUIRE_TYPE
\x12\x12\n\x0e
PYTHON_PACKAGE
\x10\x00\x12\x0e\n\n
HUB_MODULE
\x10\x01\x12\n\n\x06
SYSTEM
\x10\x02\x12\x0b\n\x07\x43
OMMAND
\x10\x03\x12\x0e\n\n
PY_VERSION
\x10\x04\x42\x02
H
\x03\x62\x06
proto3'
))
_sym_db
.
RegisterFileDescriptor
(
DESCRIPTOR
)
...
...
@@ -35,8 +35,8 @@ _FILE_TYPE = _descriptor.EnumDescriptor(
],
containing_type
=
None
,
options
=
None
,
serialized_start
=
5
51
,
serialized_end
=
5
81
,
serialized_start
=
5
22
,
serialized_end
=
5
52
,
)
_sym_db
.
RegisterEnumDescriptor
(
_FILE_TYPE
)
...
...
@@ -60,8 +60,8 @@ _REQUIRE_TYPE = _descriptor.EnumDescriptor(
],
containing_type
=
None
,
options
=
None
,
serialized_start
=
5
83
,
serialized_end
=
6
74
,
serialized_start
=
5
54
,
serialized_end
=
6
45
,
)
_sym_db
.
RegisterEnumDescriptor
(
_REQUIRE_TYPE
)
...
...
@@ -346,22 +346,6 @@ _CHECKINFO = _descriptor.Descriptor(
is_extension
=
False
,
extension_scope
=
None
,
options
=
None
),
_descriptor
.
FieldDescriptor
(
name
=
'module_code_version'
,
full_name
=
'paddlehub.module.checkinfo.CheckInfo.module_code_version'
,
index
=
5
,
number
=
6
,
type
=
9
,
cpp_type
=
9
,
label
=
1
,
has_default_value
=
False
,
default_value
=
_b
(
""
).
decode
(
'utf-8'
),
message_type
=
None
,
enum_type
=
None
,
containing_type
=
None
,
is_extension
=
False
,
extension_scope
=
None
,
options
=
None
),
],
extensions
=
[],
nested_types
=
[],
...
...
@@ -372,7 +356,7 @@ _CHECKINFO = _descriptor.Descriptor(
extension_ranges
=
[],
oneofs
=
[],
serialized_start
=
320
,
serialized_end
=
5
49
,
serialized_end
=
5
20
,
)
_FILEINFO
.
fields_by_name
[
'type'
].
enum_type
=
_FILE_TYPE
...
...
paddlehub/module/manager.py
浏览文件 @
eb0785c1
...
...
@@ -22,6 +22,9 @@ import shutil
from
functools
import
cmp_to_key
import
tarfile
import
sys
import
importlib
import
inspect
import
paddlehub
as
hub
from
paddlehub.common
import
utils
...
...
@@ -54,6 +57,22 @@ class LocalModuleManager(object):
"version"
].
s
return
True
,
info
else
:
module_file
=
os
.
path
.
join
(
module_path
,
'module.py'
)
if
os
.
path
.
exists
(
module_file
):
basename
=
os
.
path
.
split
(
module_path
)[
-
1
]
dirname
=
os
.
path
.
join
(
*
list
(
os
.
path
.
split
(
module_path
)[:
-
1
]))
sys
.
path
.
insert
(
0
,
dirname
)
_module
=
importlib
.
import_module
(
"{}.module"
.
format
(
basename
))
for
_item
,
_cls
in
inspect
.
getmembers
(
_module
,
inspect
.
isclass
):
_item
=
_module
.
__dict__
[
_item
]
if
issubclass
(
_item
,
hub
.
Module
):
version
=
_item
.
_version
break
sys
.
path
.
pop
(
0
)
return
True
,
{
'version'
:
version
}
logger
.
warning
(
"%s does not exist, the module will be reinstalled"
%
desc_pb_path
)
...
...
paddlehub/module/module.py
浏览文件 @
eb0785c1
...
...
@@ -54,72 +54,6 @@ PYTHON_DIR = "python"
PROCESSOR_NAME
=
"processor"
# PaddleHub var prefix
HUB_VAR_PREFIX
=
"@HUB_%s@"
# PaddleHub Module package suffix
HUB_PACKAGE_SUFFIX
=
"phm"
def
create_module
(
directory
,
name
,
author
,
email
,
module_type
,
summary
,
version
):
save_file
=
"{}-{}.{}"
.
format
(
name
,
version
,
HUB_PACKAGE_SUFFIX
)
with
tmp_dir
()
as
base_dir
:
# package the module
with
tarfile
.
open
(
save_file
,
"w:gz"
)
as
tar
:
module_dir
=
os
.
path
.
join
(
base_dir
,
name
)
shutil
.
copytree
(
directory
,
module_dir
)
# record module info and serialize
desc
=
module_desc_pb2
.
ModuleDesc
()
attr
=
desc
.
attr
attr
.
type
=
module_desc_pb2
.
MAP
module_info
=
attr
.
map
.
data
[
'module_info'
]
module_info
.
type
=
module_desc_pb2
.
MAP
utils
.
from_pyobj_to_module_attr
(
name
,
module_info
.
map
.
data
[
'name'
])
utils
.
from_pyobj_to_module_attr
(
author
,
module_info
.
map
.
data
[
'author'
])
utils
.
from_pyobj_to_module_attr
(
email
,
module_info
.
map
.
data
[
'author_email'
])
utils
.
from_pyobj_to_module_attr
(
module_type
,
module_info
.
map
.
data
[
'type'
])
utils
.
from_pyobj_to_module_attr
(
summary
,
module_info
.
map
.
data
[
'summary'
])
utils
.
from_pyobj_to_module_attr
(
version
,
module_info
.
map
.
data
[
'version'
])
module_desc_path
=
os
.
path
.
join
(
module_dir
,
"module_desc.pb"
)
with
open
(
module_desc_path
,
"wb"
)
as
f
:
f
.
write
(
desc
.
SerializeToString
())
# generate check info
checker
=
ModuleChecker
(
module_dir
)
checker
.
generate_check_info
()
# add __init__
module_init
=
os
.
path
.
join
(
module_dir
,
"__init__.py"
)
with
open
(
module_init
,
"a"
)
as
file
:
file
.
write
(
""
)
_cwd
=
os
.
getcwd
()
os
.
chdir
(
base_dir
)
module_dir
=
module_dir
.
replace
(
base_dir
,
"."
)
tar
.
add
(
module_dir
,
recursive
=
False
)
files
=
[]
for
dirname
,
_
,
subfiles
in
os
.
walk
(
module_dir
):
for
file
in
subfiles
:
# if file.startswith("."):
# continue
files
.
append
(
os
.
path
.
join
(
dirname
,
file
))
total_length
=
len
(
files
)
print
(
"Create Module {}-{}"
.
format
(
name
,
version
))
for
index
,
file
in
enumerate
(
files
):
done
=
int
(
float
(
index
)
/
total_length
*
50
)
progress
(
"[%-50s] %.2f%%"
%
(
'='
*
done
,
float
(
index
/
total_length
*
100
)))
tar
.
add
(
file
)
progress
(
"[%-50s] %.2f%%"
%
(
'='
*
50
,
100
),
end
=
True
)
print
(
"Module package saved as {}"
.
format
(
save_file
))
os
.
chdir
(
_cwd
)
_module_runnable_func
=
{}
...
...
@@ -147,10 +81,22 @@ def serving(func):
return
_wrapper
class
Module
(
object
):
def
moduleinfo
(
name
,
version
,
author
,
author_email
,
summary
,
type
):
def
_wrapper
(
cls
):
if
not
issubclass
(
cls
,
Module
):
raise
RuntimeError
cls
.
_name
=
name
cls
.
_version
=
version
cls
.
_author
=
author
cls
.
_author_email
=
author_email
cls
.
_summary
=
summary
cls
.
_type
=
type
return
cls
return
_wrapper
_record
=
{}
class
Module
(
object
):
def
__new__
(
cls
,
name
=
None
,
directory
=
None
,
...
...
@@ -175,6 +121,12 @@ class Module(object):
directory
=
module_dir
module
=
cls
.
init_with_directory
(
directory
=
directory
,
**
kwargs
)
CacheUpdater
(
"update_cache"
,
module
.
name
,
module
.
version
).
start
()
else
:
if
not
name
and
not
directory
:
directory
=
os
.
path
.
dirname
(
os
.
path
.
abspath
(
sys
.
modules
[
cls
.
__module__
].
__file__
))
module
=
Module
.
init_with_directory
(
directory
=
directory
,
**
kwargs
)
else
:
module
=
object
.
__new__
(
cls
)
...
...
@@ -187,9 +139,8 @@ class Module(object):
version
=
None
,
**
kwargs
):
# Avoid module being initialized multiple times
if
not
directory
or
id
(
self
)
in
Module
.
_record
:
if
"_is_initialize"
in
self
.
__dict__
and
self
.
_is_initialize
:
return
Module
.
_record
[
id
(
self
)]
=
True
mod
=
self
.
__class__
.
__module__
+
"."
+
self
.
__class__
.
__name__
if
mod
in
_module_runnable_func
:
...
...
@@ -200,26 +151,8 @@ class Module(object):
self
.
_serving_func_name
=
_module_serving_func
.
get
(
mod
,
None
)
self
.
_code_version
=
"v2"
self
.
_directory
=
directory
self
.
module_desc_path
=
os
.
path
.
join
(
self
.
directory
,
MODULE_DESC_PBNAME
)
self
.
_desc
=
module_desc_pb2
.
ModuleDesc
()
with
open
(
self
.
module_desc_path
,
"rb"
)
as
file
:
self
.
_desc
.
ParseFromString
(
file
.
read
())
module_info
=
self
.
desc
.
attr
.
map
.
data
[
'module_info'
]
self
.
_name
=
utils
.
from_module_attr_to_pyobj
(
module_info
.
map
.
data
[
'name'
])
self
.
_author
=
utils
.
from_module_attr_to_pyobj
(
module_info
.
map
.
data
[
'author'
])
self
.
_author_email
=
utils
.
from_module_attr_to_pyobj
(
module_info
.
map
.
data
[
'author_email'
])
self
.
_version
=
utils
.
from_module_attr_to_pyobj
(
module_info
.
map
.
data
[
'version'
])
self
.
_type
=
utils
.
from_module_attr_to_pyobj
(
module_info
.
map
.
data
[
'type'
])
self
.
_summary
=
utils
.
from_module_attr_to_pyobj
(
module_info
.
map
.
data
[
'summary'
])
self
.
_initialize
(
**
kwargs
)
self
.
_is_initialize
=
True
@
classmethod
def
init_with_name
(
cls
,
name
,
version
=
None
,
**
kwargs
):
...
...
@@ -243,16 +176,15 @@ class Module(object):
@
classmethod
def
init_with_directory
(
cls
,
directory
,
**
kwargs
):
desc_file
=
os
.
path
.
join
(
directory
,
MODULE_DESC_PBNAME
)
if
os
.
path
.
exists
(
desc_file
):
checker
=
ModuleChecker
(
directory
)
checker
.
check
()
return
ModuleV1
(
directory
=
directory
,
**
kwargs
)
module_code_version
=
checker
.
module_code_version
if
module_code_version
==
"v2"
:
sys
.
path
.
insert
(
0
,
directory
)
# clear module cache
if
'module'
in
sys
.
modules
:
sys
.
modules
.
pop
(
'module'
)
_module
=
importlib
.
import_module
(
"module"
)
basename
=
os
.
path
.
split
(
directory
)[
-
1
]
dirname
=
os
.
path
.
join
(
*
list
(
os
.
path
.
split
(
directory
)[:
-
1
]))
sys
.
path
.
insert
(
0
,
dirname
)
_module
=
importlib
.
import_module
(
"{}.module"
.
format
(
basename
))
for
_item
,
_cls
in
inspect
.
getmembers
(
_module
,
inspect
.
isclass
):
_item
=
_module
.
__dict__
[
_item
]
if
issubclass
(
_item
,
Module
):
...
...
@@ -260,47 +192,38 @@ class Module(object):
break
sys
.
path
.
pop
(
0
)
return
user_module
return
ModuleV1
(
directory
=
directory
,
**
kwargs
)
@
property
def
run_func
(
self
):
return
self
.
_run_func
@
property
def
desc
(
self
):
return
self
.
_desc
@
property
def
directory
(
self
):
return
self
.
_directory
@
property
def
author
(
self
):
return
self
.
_author
return
self
.
_
_class__
.
_
author
@
property
def
author_email
(
self
):
return
self
.
_author_email
return
self
.
_
_class__
.
_
author_email
@
property
def
summary
(
self
):
return
self
.
_summary
return
self
.
_
_class__
.
_
summary
@
property
def
type
(
self
):
return
self
.
_type
return
self
.
_
_class__
.
_
type
@
property
def
version
(
self
):
return
self
.
_version
return
self
.
_
_class__
.
_
version
@
property
def
name
(
self
):
return
self
.
_name
@
property
def
code_version
(
self
):
return
self
.
_code_version
return
self
.
__class__
.
_name
@
property
def
is_runnable
(
self
):
...
...
@@ -340,7 +263,6 @@ class ModuleV1(Module):
if
not
directory
:
return
super
(
ModuleV1
,
self
).
__init__
(
name
,
directory
,
module_dir
,
version
)
self
.
_code_version
=
"v1"
self
.
program
=
None
self
.
assets
=
[]
self
.
helper
=
None
...
...
@@ -349,6 +271,26 @@ class ModuleV1(Module):
self
.
processor
=
None
self
.
extra_info
=
{}
# parse desc
self
.
module_desc_path
=
os
.
path
.
join
(
self
.
directory
,
MODULE_DESC_PBNAME
)
self
.
_desc
=
module_desc_pb2
.
ModuleDesc
()
with
open
(
self
.
module_desc_path
,
"rb"
)
as
file
:
self
.
_desc
.
ParseFromString
(
file
.
read
())
module_info
=
self
.
desc
.
attr
.
map
.
data
[
'module_info'
]
self
.
_name
=
utils
.
from_module_attr_to_pyobj
(
module_info
.
map
.
data
[
'name'
])
self
.
_author
=
utils
.
from_module_attr_to_pyobj
(
module_info
.
map
.
data
[
'author'
])
self
.
_author_email
=
utils
.
from_module_attr_to_pyobj
(
module_info
.
map
.
data
[
'author_email'
])
self
.
_version
=
utils
.
from_module_attr_to_pyobj
(
module_info
.
map
.
data
[
'version'
])
self
.
_type
=
utils
.
from_module_attr_to_pyobj
(
module_info
.
map
.
data
[
'type'
])
self
.
_summary
=
utils
.
from_module_attr_to_pyobj
(
module_info
.
map
.
data
[
'summary'
])
# cache data
self
.
last_call_name
=
None
self
.
cache_feed_dict
=
None
...
...
@@ -376,6 +318,33 @@ class ModuleV1(Module):
serving_func_name
=
self
.
desc
.
attr
.
map
.
data
[
'default_signature'
].
s
return
serving_func_name
if
serving_func_name
!=
""
else
None
def
desc
(
self
):
return
self
.
_desc
@
property
def
author
(
self
):
return
self
.
_author
@
property
def
author_email
(
self
):
return
self
.
_author_email
@
property
def
summary
(
self
):
return
self
.
_summary
@
property
def
type
(
self
):
return
self
.
_type
@
property
def
version
(
self
):
return
self
.
_version
@
property
def
name
(
self
):
return
self
.
_name
def
_dump_processor
(
self
):
import
inspect
pymodule
=
inspect
.
getmodule
(
self
.
processor
)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录