Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
PaddlePaddle
PaddleOCR
提交
c3924a95
P
PaddleOCR
项目概览
PaddlePaddle
/
PaddleOCR
大约 1 年 前同步成功
通知
1528
Star
32962
Fork
6643
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
108
列表
看板
标记
里程碑
合并请求
7
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
PaddleOCR
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
108
Issue
108
列表
看板
标记
里程碑
合并请求
7
合并请求
7
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
c3924a95
编写于
8月 22, 2022
作者:
文幕地方
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
add amp eval
上级
0a247f02
变更
3
显示空白变更内容
内联
并排
Showing
3 changed file
with
43 addition
and
13 deletion
+43
-13
tools/eval.py
tools/eval.py
+26
-3
tools/program.py
tools/program.py
+10
-6
tools/train.py
tools/train.py
+7
-4
未找到文件。
tools/eval.py
浏览文件 @
c3924a95
...
@@ -23,6 +23,7 @@ __dir__ = os.path.dirname(os.path.abspath(__file__))
...
@@ -23,6 +23,7 @@ __dir__ = os.path.dirname(os.path.abspath(__file__))
sys
.
path
.
insert
(
0
,
__dir__
)
sys
.
path
.
insert
(
0
,
__dir__
)
sys
.
path
.
insert
(
0
,
os
.
path
.
abspath
(
os
.
path
.
join
(
__dir__
,
'..'
)))
sys
.
path
.
insert
(
0
,
os
.
path
.
abspath
(
os
.
path
.
join
(
__dir__
,
'..'
)))
import
paddle
from
ppocr.data
import
build_dataloader
from
ppocr.data
import
build_dataloader
from
ppocr.modeling.architectures
import
build_model
from
ppocr.modeling.architectures
import
build_model
from
ppocr.postprocess
import
build_post_process
from
ppocr.postprocess
import
build_post_process
...
@@ -86,6 +87,30 @@ def main():
...
@@ -86,6 +87,30 @@ def main():
else
:
else
:
model_type
=
None
model_type
=
None
# build metric
eval_class
=
build_metric
(
config
[
'Metric'
])
# amp
use_amp
=
config
[
"Global"
].
get
(
"use_amp"
,
False
)
amp_level
=
config
[
"Global"
].
get
(
"amp_level"
,
'O2'
)
amp_custom_black_list
=
config
[
'Global'
].
get
(
'amp_custom_black_list'
,[])
if
use_amp
:
AMP_RELATED_FLAGS_SETTING
=
{
'FLAGS_cudnn_batchnorm_spatial_persistent'
:
1
,
'FLAGS_max_inplace_grad_add'
:
8
,
}
paddle
.
fluid
.
set_flags
(
AMP_RELATED_FLAGS_SETTING
)
scale_loss
=
config
[
"Global"
].
get
(
"scale_loss"
,
1.0
)
use_dynamic_loss_scaling
=
config
[
"Global"
].
get
(
"use_dynamic_loss_scaling"
,
False
)
scaler
=
paddle
.
amp
.
GradScaler
(
init_loss_scaling
=
scale_loss
,
use_dynamic_loss_scaling
=
use_dynamic_loss_scaling
)
if
amp_level
==
"O2"
:
model
=
paddle
.
amp
.
decorate
(
models
=
model
,
level
=
amp_level
,
master_weight
=
True
)
else
:
scaler
=
None
best_model_dict
=
load_model
(
best_model_dict
=
load_model
(
config
,
model
,
model_type
=
config
[
'Architecture'
][
"model_type"
])
config
,
model
,
model_type
=
config
[
'Architecture'
][
"model_type"
])
if
len
(
best_model_dict
):
if
len
(
best_model_dict
):
...
@@ -93,11 +118,9 @@ def main():
...
@@ -93,11 +118,9 @@ def main():
for
k
,
v
in
best_model_dict
.
items
():
for
k
,
v
in
best_model_dict
.
items
():
logger
.
info
(
'{}:{}'
.
format
(
k
,
v
))
logger
.
info
(
'{}:{}'
.
format
(
k
,
v
))
# build metric
eval_class
=
build_metric
(
config
[
'Metric'
])
# start eval
# start eval
metric
=
program
.
eval
(
model
,
valid_dataloader
,
post_process_class
,
metric
=
program
.
eval
(
model
,
valid_dataloader
,
post_process_class
,
eval_class
,
model_type
,
extra_input
)
eval_class
,
model_type
,
extra_input
,
scaler
,
amp_level
,
amp_custom_black_list
)
logger
.
info
(
'metric eval ***************'
)
logger
.
info
(
'metric eval ***************'
)
for
k
,
v
in
metric
.
items
():
for
k
,
v
in
metric
.
items
():
logger
.
info
(
'{}:{}'
.
format
(
k
,
v
))
logger
.
info
(
'{}:{}'
.
format
(
k
,
v
))
...
...
tools/program.py
浏览文件 @
c3924a95
...
@@ -191,7 +191,8 @@ def train(config,
...
@@ -191,7 +191,8 @@ def train(config,
logger
,
logger
,
log_writer
=
None
,
log_writer
=
None
,
scaler
=
None
,
scaler
=
None
,
amp_level
=
'O2'
):
amp_level
=
'O2'
,
amp_custom_black_list
=
[]):
cal_metric_during_train
=
config
[
'Global'
].
get
(
'cal_metric_during_train'
,
cal_metric_during_train
=
config
[
'Global'
].
get
(
'cal_metric_during_train'
,
False
)
False
)
calc_epoch_interval
=
config
[
'Global'
].
get
(
'calc_epoch_interval'
,
1
)
calc_epoch_interval
=
config
[
'Global'
].
get
(
'calc_epoch_interval'
,
1
)
...
@@ -277,8 +278,7 @@ def train(config,
...
@@ -277,8 +278,7 @@ def train(config,
model_average
=
True
model_average
=
True
# use amp
# use amp
if
scaler
:
if
scaler
:
custom_black_list
=
config
[
'Global'
].
get
(
'amp_custom_black_list'
,[])
with
paddle
.
amp
.
auto_cast
(
level
=
amp_level
,
custom_black_list
=
amp_custom_black_list
):
with
paddle
.
amp
.
auto_cast
(
level
=
amp_level
,
custom_black_list
=
custom_black_list
):
if
model_type
==
'table'
or
extra_input
:
if
model_type
==
'table'
or
extra_input
:
preds
=
model
(
images
,
data
=
batch
[
1
:])
preds
=
model
(
images
,
data
=
batch
[
1
:])
elif
model_type
in
[
"kie"
,
'vqa'
]:
elif
model_type
in
[
"kie"
,
'vqa'
]:
...
@@ -383,7 +383,9 @@ def train(config,
...
@@ -383,7 +383,9 @@ def train(config,
eval_class
,
eval_class
,
model_type
,
model_type
,
extra_input
=
extra_input
,
extra_input
=
extra_input
,
scaler
=
scaler
)
scaler
=
scaler
,
amp_level
=
amp_level
,
amp_custom_black_list
=
amp_custom_black_list
)
cur_metric_str
=
'cur metric, {}'
.
format
(
', '
.
join
(
cur_metric_str
=
'cur metric, {}'
.
format
(
', '
.
join
(
[
'{}: {}'
.
format
(
k
,
v
)
for
k
,
v
in
cur_metric
.
items
()]))
[
'{}: {}'
.
format
(
k
,
v
)
for
k
,
v
in
cur_metric
.
items
()]))
logger
.
info
(
cur_metric_str
)
logger
.
info
(
cur_metric_str
)
...
@@ -474,7 +476,9 @@ def eval(model,
...
@@ -474,7 +476,9 @@ def eval(model,
eval_class
,
eval_class
,
model_type
=
None
,
model_type
=
None
,
extra_input
=
False
,
extra_input
=
False
,
scaler
=
None
):
scaler
=
None
,
amp_level
=
'O2'
,
amp_custom_black_list
=
[]):
model
.
eval
()
model
.
eval
()
with
paddle
.
no_grad
():
with
paddle
.
no_grad
():
total_frame
=
0.0
total_frame
=
0.0
...
@@ -495,7 +499,7 @@ def eval(model,
...
@@ -495,7 +499,7 @@ def eval(model,
# use amp
# use amp
if
scaler
:
if
scaler
:
with
paddle
.
amp
.
auto_cast
(
level
=
'O2'
):
with
paddle
.
amp
.
auto_cast
(
level
=
amp_level
,
custom_black_list
=
amp_custom_black_list
):
if
model_type
==
'table'
or
extra_input
:
if
model_type
==
'table'
or
extra_input
:
preds
=
model
(
images
,
data
=
batch
[
1
:])
preds
=
model
(
images
,
data
=
batch
[
1
:])
elif
model_type
in
[
"kie"
,
'vqa'
]:
elif
model_type
in
[
"kie"
,
'vqa'
]:
...
...
tools/train.py
浏览文件 @
c3924a95
...
@@ -138,9 +138,7 @@ def main(config, device, logger, vdl_writer):
...
@@ -138,9 +138,7 @@ def main(config, device, logger, vdl_writer):
# build metric
# build metric
eval_class
=
build_metric
(
config
[
'Metric'
])
eval_class
=
build_metric
(
config
[
'Metric'
])
# load pretrain model
pre_best_model_dict
=
load_model
(
config
,
model
,
optimizer
,
config
[
'Architecture'
][
"model_type"
])
logger
.
info
(
'train dataloader has {} iters'
.
format
(
len
(
train_dataloader
)))
logger
.
info
(
'train dataloader has {} iters'
.
format
(
len
(
train_dataloader
)))
if
valid_dataloader
is
not
None
:
if
valid_dataloader
is
not
None
:
logger
.
info
(
'valid dataloader has {} iters'
.
format
(
logger
.
info
(
'valid dataloader has {} iters'
.
format
(
...
@@ -148,6 +146,7 @@ def main(config, device, logger, vdl_writer):
...
@@ -148,6 +146,7 @@ def main(config, device, logger, vdl_writer):
use_amp
=
config
[
"Global"
].
get
(
"use_amp"
,
False
)
use_amp
=
config
[
"Global"
].
get
(
"use_amp"
,
False
)
amp_level
=
config
[
"Global"
].
get
(
"amp_level"
,
'O2'
)
amp_level
=
config
[
"Global"
].
get
(
"amp_level"
,
'O2'
)
amp_custom_black_list
=
config
[
'Global'
].
get
(
'amp_custom_black_list'
,[])
if
use_amp
:
if
use_amp
:
AMP_RELATED_FLAGS_SETTING
=
{
AMP_RELATED_FLAGS_SETTING
=
{
'FLAGS_cudnn_batchnorm_spatial_persistent'
:
1
,
'FLAGS_cudnn_batchnorm_spatial_persistent'
:
1
,
...
@@ -166,12 +165,16 @@ def main(config, device, logger, vdl_writer):
...
@@ -166,12 +165,16 @@ def main(config, device, logger, vdl_writer):
else
:
else
:
scaler
=
None
scaler
=
None
# load pretrain model
pre_best_model_dict
=
load_model
(
config
,
model
,
optimizer
,
config
[
'Architecture'
][
"model_type"
])
if
config
[
'Global'
][
'distributed'
]:
if
config
[
'Global'
][
'distributed'
]:
model
=
paddle
.
DataParallel
(
model
)
model
=
paddle
.
DataParallel
(
model
)
# start train
# start train
program
.
train
(
config
,
train_dataloader
,
valid_dataloader
,
device
,
model
,
program
.
train
(
config
,
train_dataloader
,
valid_dataloader
,
device
,
model
,
loss_class
,
optimizer
,
lr_scheduler
,
post_process_class
,
loss_class
,
optimizer
,
lr_scheduler
,
post_process_class
,
eval_class
,
pre_best_model_dict
,
logger
,
vdl_writer
,
scaler
,
amp_level
)
eval_class
,
pre_best_model_dict
,
logger
,
vdl_writer
,
scaler
,
amp_level
,
amp_custom_black_list
)
def
test_reader
(
config
,
device
,
logger
):
def
test_reader
(
config
,
device
,
logger
):
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录