Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
PaddlePaddle
PaddleClas
提交
0e5cbd2b
P
PaddleClas
项目概览
PaddlePaddle
/
PaddleClas
大约 1 年 前同步成功
通知
115
Star
4999
Fork
1114
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
19
列表
看板
标记
里程碑
合并请求
6
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
PaddleClas
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
19
Issue
19
列表
看板
标记
里程碑
合并请求
6
合并请求
6
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
0e5cbd2b
编写于
11月 21, 2022
作者:
X
xiongkun
提交者:
Wei Shengyu
11月 22, 2022
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
[dy2static-tipc] add txt config for dy2static test
上级
f04eb47f
变更
4
隐藏空白更改
内联
并排
Showing
4 changed file
with
186 addition
and
41 deletion
+186
-41
test_tipc/configs/MobileNetV1/MobileNetV1_train_dy2static_python.txt
...onfigs/MobileNetV1/MobileNetV1_train_dy2static_python.txt
+44
-0
test_tipc/configs/MobileNetV2/MobileNetV2_train_dy2static_python.txt
...onfigs/MobileNetV2/MobileNetV2_train_dy2static_python.txt
+44
-0
test_tipc/configs/MobileNetV3/MobileNetV3_large_x1_0_train_dy2static_python.txt
...leNetV3/MobileNetV3_large_x1_0_train_dy2static_python.txt
+44
-0
test_tipc/test_train_dy2static_python.sh
test_tipc/test_train_dy2static_python.sh
+54
-41
未找到文件。
test_tipc/configs/MobileNetV1/MobileNetV1_train_dy2static_python.txt
0 → 100644
浏览文件 @
0e5cbd2b
=========================== base_train ===========================
model_name:MobileNetV2
python:python3.7
gpu_list:0
-o Global.device:gpu
-o Global.auto_cast:null
-o Global.epochs:lite_train_lite_infer=2|whole_train_whole_infer=120
-o Global.output_dir:./output/
-o DataLoader.Train.sampler.batch_size:8
-o Global.pretrained_model:null
train_model_name:latest
train_infer_img_dir:./dataset/ILSVRC2012/val
null:null
##
trainer:to_static_train
norm_train:tools/train.py -c ppcls/configs/ImageNet/MobileNetV2/MobileNetV2.yaml -o Global.seed=1234 -o DataLoader.Train.sampler.shuffle=False -o DataLoader.Train.loader.num_workers=0 -o DataLoader.Train.loader.use_shared_memory=False -o Global.eval_during_train=False -o Global.save_interval=2 -o Global.print_batch_step=1
pact_train:null
fpgm_train:null
distill_train:null
to_static_train:-o Global.to_static=True
null:null
##
=========================== amp_train ===========================
model_name:MobileNetV1
python:python3.7
gpu_list:0
-o Global.device:gpu
-o Global.auto_cast:null
-o Global.epochs:lite_train_lite_infer=2|whole_train_whole_infer=120
-o Global.output_dir:./output/
-o DataLoader.Train.sampler.batch_size:8
-o Global.pretrained_model:null
train_model_name:latest
train_infer_img_dir:./dataset/ILSVRC2012/val
null:null
##
trainer:to_static_train
amp_train:tools/train.py -c ppcls/configs/ImageNet/MobileNetV1/MobileNetV1.yaml -o Global.seed=1234 -o DataLoader.Train.sampler.shuffle=False -o DataLoader.Train.loader.num_workers=0 -o DataLoader.Train.loader.use_shared_memory=False -o AMP.scale_loss=128 -o AMP.use_dynamic_loss_scaling=True -o AMP.level=O2 -o Global.eval_during_train=False -o Global.save_interval=2 -o Global.print_batch_step=1
pact_train:null
fpgm_train:null
distill_train:null
to_static_train:-o Global.to_static=True
null:null
##
test_tipc/configs/MobileNetV2/MobileNetV2_train_dy2static_python.txt
0 → 100644
浏览文件 @
0e5cbd2b
=========================== base_train ===========================
model_name:MobileNetV2
python:python3.7
gpu_list:0
-o Global.device:gpu
-o Global.auto_cast:null
-o Global.epochs:lite_train_lite_infer=2|whole_train_whole_infer=120
-o Global.output_dir:./output/
-o DataLoader.Train.sampler.batch_size:8
-o Global.pretrained_model:null
train_model_name:latest
train_infer_img_dir:./dataset/ILSVRC2012/val
null:null
##
trainer:to_static_train
norm_train:tools/train.py -c ppcls/configs/ImageNet/MobileNetV2/MobileNetV2.yaml -o Global.seed=1234 -o DataLoader.Train.sampler.shuffle=False -o DataLoader.Train.loader.num_workers=0 -o DataLoader.Train.loader.use_shared_memory=False -o Global.eval_during_train=False -o Global.save_interval=2 -o Global.print_batch_step=1
pact_train:null
fpgm_train:null
distill_train:null
to_static_train:-o Global.to_static=True
null:null
##
=========================== amp_train ===========================
model_name:MobileNetV2
python:python3.7
gpu_list:0
-o Global.device:gpu
-o Global.auto_cast:null
-o Global.epochs:lite_train_lite_infer=2|whole_train_whole_infer=120
-o Global.output_dir:./output/
-o DataLoader.Train.sampler.batch_size:8
-o Global.pretrained_model:null
train_model_name:latest
train_infer_img_dir:./dataset/ILSVRC2012/val
null:null
##
trainer:to_static_train
amp_train:tools/train.py -c ppcls/configs/ImageNet/MobileNetV2/MobileNetV2.yaml -o Global.seed=1234 -o DataLoader.Train.sampler.shuffle=False -o DataLoader.Train.loader.num_workers=0 -o DataLoader.Train.loader.use_shared_memory=False -o AMP.scale_loss=128 -o AMP.use_dynamic_loss_scaling=True -o AMP.level=O2 -o Global.eval_during_train=False -o Global.save_interval=2 -o Global.print_batch_step=1
pact_train:null
fpgm_train:null
distill_train:null
to_static_train:-o Global.to_static=True
null:null
##
test_tipc/configs/MobileNetV3/MobileNetV3_large_x1_0_train_dy2static_python.txt
0 → 100644
浏览文件 @
0e5cbd2b
=========================== base_train ===========================
model_name:MobileNetV3_large_x1_0
python:python3.7
gpu_list:0
-o Global.device:cpu
-o Global.auto_cast:null
-o Global.epochs:lite_train_lite_infer=2|whole_train_whole_infer=120
-o Global.output_dir:./output/
-o DataLoader.Train.sampler.batch_size:8
-o Global.pretrained_model:null
train_model_name:latest
train_infer_img_dir:./dataset/ILSVRC2012/val
null:null
##
trainer:to_static_train
norm_train:tools/train.py -c ppcls/configs/ImageNet/MobileNetV3/MobileNetV3_large_x1_0.yaml -o Global.seed=1234 -o DataLoader.Train.sampler.shuffle=False -o DataLoader.Train.loader.num_workers=0 -o DataLoader.Train.loader.use_shared_memory=False -o Global.eval_during_train=False -o Global.save_interval=2 -o Global.print_batch_step=1 -o Global.print_batch_step=1 -o Global.print_batch_step=1 -o Global.print_batch_step=1 -o Global.print_batch_step=1 -o Global.print_batch_step=1 -o Global.print_batch_step=1 -o Global.print_batch_step=1
pact_train:null
fpgm_train:null
distill_train:null
to_static_train:-o Global.to_static=True
null:null
##
=========================== amp_train ===========================
model_name:MobileNetV3_large_x1_0
python:python3.7
gpu_list:0|0,1
-o Global.device:gpu
-o Global.auto_cast:null
-o Global.epochs:lite_train_lite_infer=2|whole_train_whole_infer=120
-o Global.output_dir:./output/
-o DataLoader.Train.sampler.batch_size:8
-o Global.pretrained_model:null
train_model_name:latest
train_infer_img_dir:./dataset/ILSVRC2012/val
null:null
##
trainer:amp_train
amp_train:tools/train.py -c ppcls/configs/ImageNet/MobileNetV3/MobileNetV3_large_x1_0.yaml -o Global.seed=1234 -o DataLoader.Train.sampler.shuffle=False -o DataLoader.Train.loader.num_workers=0 -o DataLoader.Train.loader.use_shared_memory=False -o AMP.scale_loss=128 -o AMP.use_dynamic_loss_scaling=True -o AMP.level=O2 -o Global.eval_during_train=False -o Global.save_interval=2
pact_train:null
fpgm_train:null
distill_train:null
null:null
null:null
##
test_tipc/test_train_dy2static_python.sh
浏览文件 @
0e5cbd2b
#!/bin/bash
#!/bin/bash
source
test_tipc/common_func.sh
source
test_tipc/common_func.sh
IFS
=
$'
\n
'
BASE_CONFIG_FILE
=
$1
# always use the lite_train_lite_infer mode to speed. Modify the config file.
# always use the lite_train_lite_infer mode to speed. Modify the config file.
MODE
=
lite_train_lite_infer
MODE
=
lite_train_lite_infer
BASEDIR
=
$(
dirname
"
$0
"
)
BASEDIR
=
$(
dirname
"
$0
"
)
FILENAME
=
$1
sed
-i
's/gpu_list.*$/gpu_list:0/g'
$FILENAME
sed
-i
'23,$d'
$FILENAME
#sed -i 's/-o Global.device:.*$/-o Global.device:cpu/g' $FILENAME
sed
-i
'16s/$/ -o Global.print_batch_step=1/'
${
FILENAME
}
# get the log path.
# get the log path.
IFS
=
$'
\n
'
dataline
=
$(
cat
${
BASE_CONFIG_FILE
}
)
dataline
=
$(
cat
${
FILENAME
}
)
lines
=(
${
dataline
}
)
lines
=(
${
dataline
}
)
model_name
=
$(
func_parser_value
"
${
lines
[1]
}
"
)
model_name
=
$(
func_parser_value
"
${
lines
[1]
}
"
)
LOG_PATH
=
"./test_tipc/output/
${
model_name
}
/
${
MODE
}
"
LOG_PATH
=
"./test_tipc/output/
${
model_name
}
/
${
MODE
}
"
...
@@ -25,35 +19,54 @@ status_log="${LOG_PATH}/results_python.log"
...
@@ -25,35 +19,54 @@ status_log="${LOG_PATH}/results_python.log"
# make cudnn algorithm deterministic, such as conv.
# make cudnn algorithm deterministic, such as conv.
export
FLAGS_cudnn_deterministic
=
True
export
FLAGS_cudnn_deterministic
=
True
# start dygraph train
# read the base config and parse and run the sub commands
dygraph_output
=
$LOG_PATH
/python_train_infer_dygraph_output.txt
config_line_numbers
=
`
cat
${
BASE_CONFIG_FILE
}
|
grep
-n
"============"
|
cut
-d
':'
-f1
`
dygraph_loss
=
$LOG_PATH
/dygraph_loss.txt
for
cln
in
$config_line_numbers
sed
-i
'15ctrainer:norm_train'
${
FILENAME
}
do
cmd
=
"bash test_tipc/test_train_inference_python.sh
${
FILENAME
}
$MODE
>
$dygraph_output
2>&1"
# change IFS to prevent \n is parsed as delimiter.
echo
$cmd
IFS
=
""
eval
$cmd
config_lines
=
$(
cat
${
BASE_CONFIG_FILE
}
|
sed
-n
"
${
cln
}
,
\$
p"
|
head
-n
22
)
config_name
=
`
echo
${
config_lines
}
|
grep
'====='
|
cut
-d
' '
-f2
`
# start dy2static train
FILENAME
=
$LOG_PATH
/dy2static_
$config_name
.txt
dy2static_output
=
$LOG_PATH
/python_train_infer_dy2static_output.txt
echo
"[Start dy2static]"
"
${
config_name
}
:
${
FILENAME
}
"
dy2static_loss
=
$LOG_PATH
/dy2static_loss.txt
echo
${
config_lines
}
>
$FILENAME
sed
-i
'15ctrainer:to_static_train'
${
FILENAME
}
sed
-i
's/gpu_list.*$/gpu_list:0/g'
$FILENAME
cmd
=
"bash test_tipc/test_train_inference_python.sh
${
FILENAME
}
$MODE
>
$dy2static_output
2>&1"
sed
-i
'16s/$/ -o Global.print_batch_step=1/'
${
FILENAME
}
echo
$cmd
eval
$cmd
IFS
=
$'
\n
'
# analysis and compare the losses.
dyout
=
`
cat
$dy2static_output
| python test_tipc/extract_loss.py
-v
'Iter:'
-e
'loss: {%f},'
`
# start dygraph train
stout
=
`
cat
$dygraph_output
| python test_tipc/extract_loss.py
-v
'Iter:'
-e
'loss: {%f},'
`
dygraph_output
=
$LOG_PATH
/
${
config_name
}
_python_train_infer_dygraph_output.txt
echo
$dyout
>
$dygraph_loss
dygraph_loss
=
$LOG_PATH
/
${
config_name
}
_dygraph_loss.txt
echo
$stout
>
$dy2static_loss
sed
-i
'15ctrainer:norm_train'
${
FILENAME
}
diff_log
=
$LOG_PATH
/diff_log.txt
cmd
=
"bash test_tipc/test_train_inference_python.sh
${
FILENAME
}
$MODE
>
$dygraph_output
2>&1"
diff_cmd
=
"diff -w
$dygraph_loss
$dy2static_loss
| tee
$diff_log
"
echo
$cmd
eval
$diff_cmd
eval
$cmd
last_status
=
$?
if
[
"
$dyout
"
=
""
]
;
then
# start dy2static train
status_check 2
$diff_cmd
$status_log
$model_name
$diff_log
dy2static_output
=
$LOG_PATH
/
${
config_name
}
_python_train_infer_dy2static_output.txt
fi
dy2static_loss
=
$LOG_PATH
/
${
config_name
}
_dy2static_loss.txt
if
[
"
$stout
"
=
""
]
;
then
sed
-i
'15ctrainer:to_static_train'
${
FILENAME
}
status_check 2
$diff_cmd
$status_log
$model_name
$diff_log
cmd
=
"bash test_tipc/test_train_inference_python.sh
${
FILENAME
}
$MODE
>
$dy2static_output
2>&1"
fi
echo
$cmd
status_check
$last_status
$diff_cmd
$status_log
$model_name
$diff_log
eval
$cmd
# analysis and compare the losses.
dyout
=
`
cat
$dy2static_output
| python test_tipc/extract_loss.py
-v
'Iter:'
-e
'loss: {%f},'
`
stout
=
`
cat
$dygraph_output
| python test_tipc/extract_loss.py
-v
'Iter:'
-e
'loss: {%f},'
`
echo
$dyout
>
$dygraph_loss
echo
$stout
>
$dy2static_loss
diff_log
=
$LOG_PATH
/
${
config_name
}
_diff_log.txt
diff_cmd
=
"diff -w
$dygraph_loss
$dy2static_loss
>
$diff_log
"
eval
$diff_cmd
last_status
=
$?
cat
$diff_log
if
[
"
$dyout
"
=
""
]
;
then
status_check 2
$diff_cmd
$status_log
$model_name
$diff_log
fi
if
[
"
$stout
"
=
""
]
;
then
status_check 2
$diff_cmd
$status_log
$model_name
$diff_log
fi
status_check
$last_status
$diff_cmd
$status_log
$model_name
$diff_log
done
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录