Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
weixin_41840029
PaddleOCR
提交
44ea4d86
P
PaddleOCR
项目概览
weixin_41840029
/
PaddleOCR
与 Fork 源项目一致
Fork自
PaddlePaddle / PaddleOCR
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
PaddleOCR
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
44ea4d86
编写于
2月 08, 2022
作者:
L
LDOUBLEV
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
keep model name and directory same
上级
f744089c
变更
4
显示空白变更内容
内联
并排
Showing
4 changed file
with
171 addition
and
483 deletion
+171
-483
test_tipc/benchmark_train.sh
test_tipc/benchmark_train.sh
+171
-128
test_tipc/benchmark_trainv2.sh
test_tipc/benchmark_trainv2.sh
+0
-258
test_tipc/configs/det_mv3_db_v2/train_benchmark.txt
test_tipc/configs/det_mv3_db_v2/train_benchmark.txt
+0
-40
test_tipc/configs/det_mv3_db_v2/train_infer_python.txt
test_tipc/configs/det_mv3_db_v2/train_infer_python.txt
+0
-57
未找到文件。
test_tipc/benchmark_train.sh
浏览文件 @
44ea4d86
#!/bin/bash
source
test_tipc/common_func.sh
# set env
python
=
python
export
model_branch
=
`
git symbolic-ref HEAD 2>/dev/null |
cut
-d
"/"
-f
3
`
export
model_commit
=
$(
git log|head
-n1
|awk
'{print $2}'
)
export
str_tmp
=
$(
echo
`
pip list|grep paddlepaddle-gpu|awk
-F
' '
'{print $2}'
`
)
export
frame_version
=
${
str_tmp
%%.post*
}
export
frame_commit
=
$(
echo
`
${
python
}
-c
"import paddle;print(paddle.version.commit)"
`
)
# run benchmark sh
# Usage:
# bash run_benchmark_train.sh config.txt params
# or
# bash run_benchmark_train.sh config.txt
function
func_parser_params
(){
strs
=
$1
...
...
@@ -55,30 +65,15 @@ function get_repo_name(){
}
FILENAME
=
$1
# MODE be one of ['benchmark_train']
# copy FILENAME as new
new_filename
=
"./test_tipc/benchmark_train.txt"
cmd
=
`
yes
|cp
$FILENAME
$new_filename
`
FILENAME
=
$new_filename
# MODE must be one of ['benchmark_train']
MODE
=
$2
params
=
$3
# bash test_tipc/benchmark_train.sh test_tipc/configs/det_mv3_db_v2.0/train_benchmark.txt benchmark_train dynamic_bs8_null_SingleP_DP_N1C1
IFS
=
"
\n
"
# parser params from input: modeltype_bs${bs_item}_${fp_item}_${run_process_type}_${run_mode}_${device_num}
IFS
=
"_"
params_list
=(
${
params
}
)
model_type
=
${
params_list
[0]
}
batch_size
=
${
params_list
[1]
}
batch_size
=
`
echo
${
batch_size
}
|
tr
-cd
"[0-9]"
`
precision
=
${
params_list
[2]
}
run_process_type
=
${
params_list
[3]
}
run_mode
=
${
params_list
[4]
}
device_num
=
${
params_list
[5]
}
device_num_copy
=
$device_num
IFS
=
";"
# sed batchsize and precision
func_sed_params
"
$FILENAME
"
"6"
"
$precision
"
func_sed_params
"
$FILENAME
"
"9"
"
$batch_size
"
PARAMS
=
$3
# bash test_tipc/benchmark_train.sh test_tipc/configs/det_mv3_db_v2_0/train_benchmark.txt benchmark_train dynamic_bs8_null_SingleP_DP_N1C1
IFS
=
$'
\n
'
# parser params from train_benchmark.txt
dataline
=
`
cat
$FILENAME
`
# parser params
...
...
@@ -87,31 +82,22 @@ lines=(${dataline})
model_name
=
$(
func_parser_value
"
${
lines
[1]
}
"
)
# 获取benchmark_params所在的行数
line_num
=
`
grep
-n
"benchmark_params"
$FILENAME
|
cut
-d
":"
-f
1
`
line_num
=
`
grep
-n
"
train_
benchmark_params"
$FILENAME
|
cut
-d
":"
-f
1
`
# for train log parser
line_num
=
`
expr
$line_num
+ 3
`
speed_unit_value
=
$(
func_parser_value
"
${
lines
[line_num]
}
"
)
batch_size
=
$(
func_parser_value
"
${
lines
[line_num]
}
"
)
line_num
=
`
expr
$line_num
+ 1
`
skip_steps_value
=
$(
func_parser_value
"
${
lines
[line_num]
}
"
)
fp_items
=
$(
func_parser_value
"
${
lines
[line_num]
}
"
)
line_num
=
`
expr
$line_num
+ 1
`
keyword_value
=
$(
func_parser_value
"
${
lines
[line_num]
}
"
)
echo
$keyword_value
epoch
=
$(
func_parser_value
"
${
lines
[line_num]
}
"
)
line_num
=
`
expr
$line_num
+ 1
`
convergence_key_value
=
$(
func_parser_value
"
${
lines
[line_num]
}
"
)
profile_option_key
=
$(
func_parser_key
"
${
lines
[line_num]
}
"
)
profile_option_params
=
$(
func_parser_value
"
${
lines
[line_num]
}
"
)
profile_option
=
"
${
profile_option_key
}
:
${
profile_option_params
}
"
line_num
=
`
expr
$line_num
+ 1
`
flags_value
=
$(
func_parser_value
"
${
lines
[line_num]
}
"
)
gpu_id
=
$(
set_gpu_id
$device_num
)
repo_name
=
$(
get_repo_name
)
SAVE_LOG
=
${
BENCHMARK_LOG_DIR
:-
$(
pwd
)
}
# */benchmark_log
status_log
=
"
${
SAVE_LOG
}
/benchmark_log/results.log"
# set export
# set flags
IFS
=
";"
flags_list
=(
${
flags_value
}
)
for
_flag
in
${
flags_list
[*]
}
;
do
...
...
@@ -119,23 +105,76 @@ for _flag in ${flags_list[*]}; do
eval
$cmd
done
if
[
${
precision
}
=
"null"
]
;
then
# set log_name
repo_name
=
$(
get_repo_name
)
SAVE_LOG
=
${
BENCHMARK_LOG_DIR
:-
$(
pwd
)
}
# */benchmark_log
mkdir
-p
"
${
SAVE_LOG
}
/benchmark_log/"
status_log
=
"
${
SAVE_LOG
}
/benchmark_log/results.log"
# The number of lines in which train params can be replaced.
line_python
=
3
line_gpuid
=
4
line_precision
=
6
line_epoch
=
7
line_batchsize
=
9
line_profile
=
13
line_eval_py
=
24
line_export_py
=
30
func_sed_params
"
$FILENAME
"
"
${
line_eval_py
}
"
"null"
func_sed_params
"
$FILENAME
"
"
${
line_export_py
}
"
"null"
func_sed_params
"
$FILENAME
"
"
${
line_python
}
"
"
$python
"
# if params
if
[
!
-n
"
$PARAMS
"
]
;
then
# PARAMS input is not a word.
IFS
=
"|"
batch_size_list
=(
${
batch_size
}
)
fp_items_list
=(
${
fp_items
}
)
device_num_list
=(
N1C4
)
run_mode
=
"DP"
else
# parser params from input: modeltype_bs${bs_item}_${fp_item}_${run_process_type}_${run_mode}_${device_num}
IFS
=
"_"
params_list
=(
${
PARAMS
}
)
model_type
=
${
params_list
[0]
}
batch_size
=
${
params_list
[1]
}
batch_size
=
`
echo
${
batch_size
}
|
tr
-cd
"[0-9]"
`
precision
=
${
params_list
[2]
}
run_process_type
=
${
params_list
[3]
}
run_mode
=
${
params_list
[4]
}
device_num
=
${
params_list
[5]
}
IFS
=
";"
if
[
${
precision
}
=
"null"
]
;
then
precision
=
"fp32"
fi
fi
# set env
export
model_branch
=
`
git symbolic-ref HEAD 2>/dev/null |
cut
-d
"/"
-f
3
`
export
model_commit
=
$(
git log|head
-n1
|awk
'{print $2}'
)
export
str_tmp
=
$(
echo
`
pip list|grep paddlepaddle-gpu|awk
-F
' '
'{print $2}'
`
)
export
frame_version
=
${
str_tmp
%%.post*
}
export
frame_commit
=
$(
echo
`
python
-c
"import paddle;print(paddle.version.commit)"
`
)
fp_items_list
=(
$precision
)
batch_size_list
=(
$batch_size
)
device_num_list
=(
$device_num
)
fi
IFS
=
"|"
for
batch_size
in
${
batch_size_list
[*]
}
;
do
for
precision
in
${
fp_items_list
[*]
}
;
do
for
device_num
in
${
device_num_list
[*]
}
;
do
# sed batchsize and precision
func_sed_params
"
$FILENAME
"
"
${
line_precision
}
"
"
$precision
"
func_sed_params
"
$FILENAME
"
"
${
line_batchsize
}
"
"
$MODE
=
$batch_size
"
func_sed_params
"
$FILENAME
"
"
${
line_epoch
}
"
"
$MODE
=
$epoch
"
gpu_id
=
$(
set_gpu_id
$device_num
)
if
[
${#
gpu_id
}
-le
1
]
;
then
if
[
${#
gpu_id
}
-le
1
]
;
then
run_process_type
=
"SingleP"
log_path
=
"
$SAVE_LOG
/profiling_log"
mkdir
-p
$log_path
log_name
=
"
${
repo_name
}
_
${
model_name
}
_bs
${
batch_size
}
_
${
precision
}
_
${
run_process_type
}
_
${
run_mode
}
_
${
device_num
}
_profiling"
func_sed_params
"
$FILENAME
"
"4"
"0"
# sed used gpu_id
func_sed_params
"
$FILENAME
"
"
${
line_gpuid
}
"
"0"
# sed used gpu_id
# set profile_option params
tmp
=
`
sed
-i
"
${
line_profile
}
s/.*/
${
profile_option
}
/"
"
${
FILENAME
}
"
`
# run test_train_inference_python.sh
cmd
=
"bash test_tipc/test_train_inference_python.sh
${
FILENAME
}
benchmark_train >
${
log_path
}
/
${
log_name
}
2>&1 "
echo
$cmd
eval
$cmd
...
...
@@ -148,7 +187,7 @@ if [ ${#gpu_id} -le 1 ];then
mkdir
-p
$speed_log_path
log_name
=
"
${
repo_name
}
_
${
model_name
}
_bs
${
batch_size
}
_
${
precision
}
_
${
run_process_type
}
_
${
run_mode
}
_
${
device_num
}
_log"
speed_log_name
=
"
${
repo_name
}
_
${
model_name
}
_bs
${
batch_size
}
_
${
precision
}
_
${
run_process_type
}
_
${
run_mode
}
_
${
device_num
}
_speed"
func_sed_params
"
$FILENAME
"
"13"
"null"
# sed used gpu_id
func_sed_params
"
$FILENAME
"
"
${
line_profile
}
"
"null"
# sed profile_id as null
cmd
=
"bash test_tipc/test_train_inference_python.sh
${
FILENAME
}
benchmark_train >
${
log_path
}
/
${
log_name
}
2>&1 "
echo
$cmd
job_bt
=
`
date
'+%Y%m%d%H%M%S'
`
...
...
@@ -159,32 +198,34 @@ if [ ${#gpu_id} -le 1 ];then
# parser log
_model_name
=
"
${
model_name
}
_bs
${
batch_size
}
_
${
precision
}
_
${
run_process_type
}
_
${
run_mode
}
"
cmd
=
"python
${
BENCHMARK_ROOT
}
/scripts/analysis.py --filename
${
log_path
}
/
${
log_name
}
\
cmd
=
"
${
python
}
${
BENCHMARK_ROOT
}
/scripts/analysis.py --filename
${
log_path
}
/
${
log_name
}
\
--speed_log_file '
${
speed_log_path
}
/
${
speed_log_name
}
'
\
--model_name
${
_model_name
}
\
--base_batch_size
${
batch_size
}
\
--run_mode
${
run_mode
}
\
--run_process_type
${
run_process_type
}
\
--fp_item
${
precision
}
\
--keyword
${
keyword_value
}
:
\
--skip_steps
${
skip_steps_value
}
\
--keyword ips
:
\
--skip_steps 2
\
--device_num
${
device_num
}
\
--speed_unit
${
speed_unit_value
}
\
--convergence_key
${
convergence_key_value
}
: "
--speed_unit samples/s
\
--convergence_key loss
: "
echo
$cmd
eval
$cmd
last_status
=
${
PIPESTATUS
[0]
}
status_check
$last_status
"
${
cmd
}
"
"
${
status_log
}
"
else
else
IFS
=
";"
unset_env
=
`
unset
CUDA_VISIBLE_DEVICES
`
run_process_type
=
"MultiP"
log_path
=
"
$SAVE_LOG
/train_log"
speed_log_path
=
"
$SAVE_LOG
/index"
mkdir
-p
$log_path
mkdir
-p
$speed_log_path
log_name
=
"
${
repo_name
}
_
${
model_name
}
_bs
${
batch_size
}
_
${
precision
}
_
${
run_process_type
}
_
${
run_mode
}
_
${
device_num
}
_log"
speed_log_name
=
"
${
repo_name
}
_
${
model_name
}
_bs
${
batch_size
}
_
${
precision
}
_
${
run_process_type
}
_
${
run_mode
}
_
${
device_num
}
_speed"
func_sed_params
"
$FILENAME
"
"4
"
"
$gpu_id
"
# sed used gpu_id
func_sed_params
"
$FILENAME
"
"13
"
"null"
# sed --profile_option as null
func_sed_params
"
$FILENAME
"
"
${
line_gpuid
}
"
"
$gpu_id
"
# sed used gpu_id
func_sed_params
"
$FILENAME
"
"
${
line_profile
}
"
"null"
# sed --profile_option as null
cmd
=
"bash test_tipc/test_train_inference_python.sh
${
FILENAME
}
benchmark_train >
${
log_path
}
/
${
log_name
}
2>&1 "
echo
$cmd
job_bt
=
`
date
'+%Y%m%d%H%M%S'
`
...
...
@@ -195,21 +236,23 @@ else
# parser log
_model_name
=
"
${
model_name
}
_bs
${
batch_size
}
_
${
precision
}
_
${
run_process_type
}
_
${
run_mode
}
"
cmd
=
"python
${
BENCHMARK_ROOT
}
/scripts/analysis.py --filename
${
log_path
}
/
${
log_name
}
\
cmd
=
"
${
python
}
${
BENCHMARK_ROOT
}
/scripts/analysis.py --filename
${
log_path
}
/
${
log_name
}
\
--speed_log_file '
${
speed_log_path
}
/
${
speed_log_name
}
'
\
--model_name
${
_model_name
}
\
--base_batch_size
${
batch_size
}
\
--run_mode
${
run_mode
}
\
--run_process_type
${
run_process_type
}
\
--fp_item
${
precision
}
\
--keyword
${
keyword_value
}
:
\
--skip_steps
${
skip_steps_value
}
\
--keyword ips
:
\
--skip_steps 2
\
--device_num
${
device_num
}
\
--speed_unit
${
speed_unit_value
}
\
--convergence_key
${
convergence_key_value
}
: "
--speed_unit images/s
\
--convergence_key loss
: "
echo
$cmd
eval
$cmd
last_status
=
${
PIPESTATUS
[0]
}
status_check
$last_status
"
${
cmd
}
"
"
${
status_log
}
"
fi
fi
done
done
done
\ No newline at end of file
test_tipc/benchmark_trainv2.sh
已删除
100644 → 0
浏览文件 @
f744089c
#!/bin/bash
source
test_tipc/common_func.sh
# set env
python
=
python3.7
export
model_branch
=
`
git symbolic-ref HEAD 2>/dev/null |
cut
-d
"/"
-f
3
`
export
model_commit
=
$(
git log|head
-n1
|awk
'{print $2}'
)
export
str_tmp
=
$(
echo
`
pip list|grep paddlepaddle-gpu|awk
-F
' '
'{print $2}'
`
)
export
frame_version
=
${
str_tmp
%%.post*
}
export
frame_commit
=
$(
echo
`
${
python
}
-c
"import paddle;print(paddle.version.commit)"
`
)
# run benchmark sh
# Usage:
# bash run_benchmark_train.sh config.txt params
# or
# bash run_benchmark_train.sh config.txt
function
func_parser_params
(){
strs
=
$1
IFS
=
"="
array
=(
${
strs
}
)
tmp
=
${
array
[1]
}
echo
${
tmp
}
}
function
func_sed_params
(){
filename
=
$1
line
=
$2
param_value
=
$3
params
=
`
sed
-n
"
${
line
}
p"
$filename
`
IFS
=
":"
array
=(
${
params
}
)
key
=
${
array
[0]
}
value
=
${
array
[1]
}
if
[[
$value
=
~
'benchmark_train'
]]
;
then
IFS
=
'='
_val
=(
${
value
}
)
param_value
=
"
${
_val
[0]
}
=
${
param_value
}
"
fi
new_params
=
"
${
key
}
:
${
param_value
}
"
IFS
=
";"
cmd
=
"sed -i '
${
line
}
s/.*/
${
new_params
}
/' '
${
filename
}
'"
eval
$cmd
}
function
set_gpu_id
(){
string
=
$1
_str
=
${
string
:1:6
}
IFS
=
"C"
arr
=(
${
_str
}
)
M
=
${
arr
[0]
}
P
=
${
arr
[1]
}
gn
=
`
expr
$P
- 1
`
gpu_num
=
`
expr
$gn
/
$M
`
seq
=
`
seq
-s
","
0
$gpu_num
`
echo
$seq
}
function
get_repo_name
(){
IFS
=
";"
cur_dir
=
$(
pwd
)
IFS
=
"/"
arr
=(
${
cur_dir
}
)
echo
${
arr
[-1]
}
}
FILENAME
=
$1
# copy FILENAME as new
new_filename
=
"./test_tipc/benchmark_train.txt"
cmd
=
`
yes
|cp
$FILENAME
$new_filename
`
FILENAME
=
$new_filename
# MODE must be one of ['benchmark_train']
MODE
=
$2
PARAMS
=
$3
# bash test_tipc/benchmark_train.sh test_tipc/configs/det_mv3_db_v2.0/train_benchmark.txt benchmark_train dynamic_bs8_null_SingleP_DP_N1C1
IFS
=
$'
\n
'
# parser params from train_benchmark.txt
dataline
=
`
cat
$FILENAME
`
# parser params
IFS
=
$'
\n
'
lines
=(
${
dataline
}
)
model_name
=
$(
func_parser_value
"
${
lines
[1]
}
"
)
# 获取benchmark_params所在的行数
line_num
=
`
grep
-n
"train_benchmark_params"
$FILENAME
|
cut
-d
":"
-f
1
`
# for train log parser
batch_size
=
$(
func_parser_value
"
${
lines
[line_num]
}
"
)
line_num
=
`
expr
$line_num
+ 1
`
fp_items
=
$(
func_parser_value
"
${
lines
[line_num]
}
"
)
line_num
=
`
expr
$line_num
+ 1
`
epoch
=
$(
func_parser_value
"
${
lines
[line_num]
}
"
)
line_num
=
`
expr
$line_num
+ 1
`
profile_option_key
=
$(
func_parser_key
"
${
lines
[line_num]
}
"
)
profile_option_params
=
$(
func_parser_value
"
${
lines
[line_num]
}
"
)
profile_option
=
"
${
profile_option_key
}
:
${
profile_option_params
}
"
line_num
=
`
expr
$line_num
+ 1
`
flags_value
=
$(
func_parser_value
"
${
lines
[line_num]
}
"
)
# set flags
IFS
=
";"
flags_list
=(
${
flags_value
}
)
for
_flag
in
${
flags_list
[*]
}
;
do
cmd
=
"export
${
_flag
}
"
eval
$cmd
done
# set log_name
repo_name
=
$(
get_repo_name
)
SAVE_LOG
=
${
BENCHMARK_LOG_DIR
:-
$(
pwd
)
}
# */benchmark_log
mkdir
-p
"
${
SAVE_LOG
}
/benchmark_log/"
status_log
=
"
${
SAVE_LOG
}
/benchmark_log/results.log"
# The number of lines in which train params can be replaced.
line_python
=
3
line_gpuid
=
4
line_precision
=
6
line_epoch
=
7
line_batchsize
=
9
line_profile
=
13
line_eval_py
=
24
line_export_py
=
30
func_sed_params
"
$FILENAME
"
"
${
line_eval_py
}
"
"null"
func_sed_params
"
$FILENAME
"
"
${
line_export_py
}
"
"null"
func_sed_params
"
$FILENAME
"
"
${
line_python
}
"
"
$python
"
# if params
if
[
!
-n
"
$PARAMS
"
]
;
then
# PARAMS input is not a word.
IFS
=
"|"
batch_size_list
=(
${
batch_size
}
)
fp_items_list
=(
${
fp_items
}
)
device_num_list
=(
N1C4
)
run_mode
=
"DP"
else
# parser params from input: modeltype_bs${bs_item}_${fp_item}_${run_process_type}_${run_mode}_${device_num}
IFS
=
"_"
params_list
=(
${
PARAMS
}
)
model_type
=
${
params_list
[0]
}
batch_size
=
${
params_list
[1]
}
batch_size
=
`
echo
${
batch_size
}
|
tr
-cd
"[0-9]"
`
precision
=
${
params_list
[2]
}
run_process_type
=
${
params_list
[3]
}
run_mode
=
${
params_list
[4]
}
device_num
=
${
params_list
[5]
}
IFS
=
";"
if
[
${
precision
}
=
"null"
]
;
then
precision
=
"fp32"
fi
fp_items_list
=(
$precision
)
batch_size_list
=(
$batch_size
)
device_num_list
=(
$device_num
)
fi
IFS
=
"|"
for
batch_size
in
${
batch_size_list
[*]
}
;
do
for
precision
in
${
fp_items_list
[*]
}
;
do
for
device_num
in
${
device_num_list
[*]
}
;
do
# sed batchsize and precision
func_sed_params
"
$FILENAME
"
"
${
line_precision
}
"
"
$precision
"
func_sed_params
"
$FILENAME
"
"
${
line_batchsize
}
"
"
$MODE
=
$batch_size
"
func_sed_params
"
$FILENAME
"
"
${
line_epoch
}
"
"
$MODE
=
$epoch
"
gpu_id
=
$(
set_gpu_id
$device_num
)
if
[
${#
gpu_id
}
-le
1
]
;
then
run_process_type
=
"SingleP"
log_path
=
"
$SAVE_LOG
/profiling_log"
mkdir
-p
$log_path
log_name
=
"
${
repo_name
}
_
${
model_name
}
_bs
${
batch_size
}
_
${
precision
}
_
${
run_process_type
}
_
${
run_mode
}
_
${
device_num
}
_profiling"
func_sed_params
"
$FILENAME
"
"
${
line_gpuid
}
"
"0"
# sed used gpu_id
# set profile_option params
tmp
=
`
sed
-i
"
${
line_profile
}
s/.*/
${
profile_option
}
/"
"
${
FILENAME
}
"
`
# run test_train_inference_python.sh
cmd
=
"bash test_tipc/test_train_inference_python.sh
${
FILENAME
}
benchmark_train >
${
log_path
}
/
${
log_name
}
2>&1 "
echo
$cmd
eval
$cmd
eval
"cat
${
log_path
}
/
${
log_name
}
"
# without profile
log_path
=
"
$SAVE_LOG
/train_log"
speed_log_path
=
"
$SAVE_LOG
/index"
mkdir
-p
$log_path
mkdir
-p
$speed_log_path
log_name
=
"
${
repo_name
}
_
${
model_name
}
_bs
${
batch_size
}
_
${
precision
}
_
${
run_process_type
}
_
${
run_mode
}
_
${
device_num
}
_log"
speed_log_name
=
"
${
repo_name
}
_
${
model_name
}
_bs
${
batch_size
}
_
${
precision
}
_
${
run_process_type
}
_
${
run_mode
}
_
${
device_num
}
_speed"
func_sed_params
"
$FILENAME
"
"
${
line_profile
}
"
"null"
# sed profile_id as null
cmd
=
"bash test_tipc/test_train_inference_python.sh
${
FILENAME
}
benchmark_train >
${
log_path
}
/
${
log_name
}
2>&1 "
echo
$cmd
job_bt
=
`
date
'+%Y%m%d%H%M%S'
`
eval
$cmd
job_et
=
`
date
'+%Y%m%d%H%M%S'
`
export
model_run_time
=
$((${
job_et
}
-
${
job_bt
}))
eval
"cat
${
log_path
}
/
${
log_name
}
"
# parser log
_model_name
=
"
${
model_name
}
_bs
${
batch_size
}
_
${
precision
}
_
${
run_process_type
}
_
${
run_mode
}
"
cmd
=
"
${
python
}
${
BENCHMARK_ROOT
}
/scripts/analysis.py --filename
${
log_path
}
/
${
log_name
}
\
--speed_log_file '
${
speed_log_path
}
/
${
speed_log_name
}
'
\
--model_name
${
_model_name
}
\
--base_batch_size
${
batch_size
}
\
--run_mode
${
run_mode
}
\
--run_process_type
${
run_process_type
}
\
--fp_item
${
precision
}
\
--keyword ips:
\
--skip_steps 2
\
--device_num
${
device_num
}
\
--speed_unit samples/s
\
--convergence_key loss: "
echo
$cmd
eval
$cmd
last_status
=
${
PIPESTATUS
[0]
}
status_check
$last_status
"
${
cmd
}
"
"
${
status_log
}
"
else
IFS
=
";"
unset_env
=
`
unset
CUDA_VISIBLE_DEVICES
`
run_process_type
=
"MultiP"
log_path
=
"
$SAVE_LOG
/train_log"
speed_log_path
=
"
$SAVE_LOG
/index"
mkdir
-p
$log_path
mkdir
-p
$speed_log_path
log_name
=
"
${
repo_name
}
_
${
model_name
}
_bs
${
batch_size
}
_
${
precision
}
_
${
run_process_type
}
_
${
run_mode
}
_
${
device_num
}
_log"
speed_log_name
=
"
${
repo_name
}
_
${
model_name
}
_bs
${
batch_size
}
_
${
precision
}
_
${
run_process_type
}
_
${
run_mode
}
_
${
device_num
}
_speed"
func_sed_params
"
$FILENAME
"
"
${
line_gpuid
}
"
"
$gpu_id
"
# sed used gpu_id
func_sed_params
"
$FILENAME
"
"
${
line_profile
}
"
"null"
# sed --profile_option as null
cmd
=
"bash test_tipc/test_train_inference_python.sh
${
FILENAME
}
benchmark_train >
${
log_path
}
/
${
log_name
}
2>&1 "
echo
$cmd
job_bt
=
`
date
'+%Y%m%d%H%M%S'
`
eval
$cmd
job_et
=
`
date
'+%Y%m%d%H%M%S'
`
export
model_run_time
=
$((${
job_et
}
-
${
job_bt
}))
eval
"cat
${
log_path
}
/
${
log_name
}
"
# parser log
_model_name
=
"
${
model_name
}
_bs
${
batch_size
}
_
${
precision
}
_
${
run_process_type
}
_
${
run_mode
}
"
cmd
=
"
${
python
}
${
BENCHMARK_ROOT
}
/scripts/analysis.py --filename
${
log_path
}
/
${
log_name
}
\
--speed_log_file '
${
speed_log_path
}
/
${
speed_log_name
}
'
\
--model_name
${
_model_name
}
\
--base_batch_size
${
batch_size
}
\
--run_mode
${
run_mode
}
\
--run_process_type
${
run_process_type
}
\
--fp_item
${
precision
}
\
--keyword ips:
\
--skip_steps 2
\
--device_num
${
device_num
}
\
--speed_unit images/s
\
--convergence_key loss: "
echo
$cmd
eval
$cmd
last_status
=
${
PIPESTATUS
[0]
}
status_check
$last_status
"
${
cmd
}
"
"
${
status_log
}
"
fi
done
done
done
\ No newline at end of file
test_tipc/configs/det_mv3_db_v2/train_benchmark.txt
已删除
100644 → 0
浏览文件 @
f744089c
===========================train_params===========================
model_name:det_mv3_db_v2
python:python
gpu_list:0
Global.use_gpu:True
Global.auto_cast:null
Global.epoch_num:benchmark_train=2
Global.save_model_dir:./output/
Train.loader.batch_size_per_card:benchmark_train=16
Global.pretrained_model:null
train_model_name:latest
train_infer_img_dir:null
--profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile
##
trainer:norm_train
norm_train:tools/train.py -c configs/det/det_mv3_db.yml -o Global.pretrained_model=./pretrain_models/MobileNetV3_large_x0_5_pretrained
pact_train:null
fpgm_train:null
distill_train:null
null:null
null:null
##
===========================eval_params===========================
eval:null
null:null
##
===========================infer_params===========================
Global.save_inference_dir:./output/
Global.checkpoints:
norm_export:null
===========================benchmark_params==========================
device_num:N1C1|N1C8|N4C32
run_process_type:MultiP
run_mode:DP|DP1-MP1-PP1|DP2-MP2-PP2
speed_unit:images/s
skip_steps:2
keyword:ips:
convergence_key:loss:
flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096
null:null
test_tipc/configs/det_mv3_db_v2/train_infer_python.txt
已删除
100644 → 0
浏览文件 @
f744089c
===========================train_params===========================
model_name:det_mv3_db_v2_0
python:python3.7
gpu_list:0|0,1
Global.use_gpu:True|True
Global.auto_cast:null
Global.epoch_num:lite_train_lite_infer=1|whole_train_whole_infer=300
Global.save_model_dir:./output/
Train.loader.batch_size_per_card:lite_train_lite_infer=2|whole_train_whole_infer=4
Global.pretrained_model:null
train_model_name:latest
train_infer_img_dir:./train_data/icdar2015/text_localization/ch4_test_images/
null:null
##
trainer:norm_train
norm_train:tools/train.py -c configs/det/det_mv3_db.yml -o Global.pretrained_model=./pretrain_models/MobileNetV3_large_x0_5_pretrained
pact_train:null
fpgm_train:null
distill_train:null
null:null
null:null
##
===========================eval_params===========================
eval:null
null:null
##
===========================infer_params===========================
Global.save_inference_dir:./output/
Global.checkpoints:
norm_export:tools/export_model.py -c configs/det/det_mv3_db.yml -o
quant_export:null
fpgm_export:null
distill_export:null
export1:null
export2:null
inference_dir:null
train_model:./inference/det_mv3_db_v2.0_train/best_accuracy
infer_export:tools/export_model.py -c configs/det/det_mv3_db.yml -o
infer_quant:False
inference:tools/infer/predict_det.py
--use_gpu:True|False
--enable_mkldnn:True|False
--cpu_threads:1|6
--rec_batch_num:1
--use_tensorrt:False|True
--precision:fp32|fp16|int8
--det_model_dir:
--image_dir:./inference/ch_det_data_50/all-sum-510/
null:null
--benchmark:True
null:null
===========================train_benchmark_params==========================
batch_size:8|16
fp_items:fp32|fp16
epoch:2
--profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile
flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096
\ No newline at end of file
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录