Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
飞不过沧海
Paddle
提交
e63a68fe
P
Paddle
项目概览
飞不过沧海
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
e63a68fe
编写于
12月 20, 2020
作者:
C
chalsliu
提交者:
GitHub
12月 20, 2020
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Retry when download failed for precision test
上级
07790ba1
变更
4
隐藏空白更改
内联
并排
Showing
4 changed file
with
83 addition
and
32 deletion
+83
-32
paddle/scripts/paddle_build.sh
paddle/scripts/paddle_build.sh
+9
-3
tools/check_added_ut.sh
tools/check_added_ut.sh
+8
-5
tools/coverage/paddle_coverage.sh
tools/coverage/paddle_coverage.sh
+10
-10
tools/get_pr_ut.py
tools/get_pr_ut.py
+56
-14
未找到文件。
paddle/scripts/paddle_build.sh
浏览文件 @
e63a68fe
...
...
@@ -1044,12 +1044,17 @@ function parallel_test_base_gpu() {
========================================
EOF
set
+
x
set
-
x
precison_cases
=
""
bash
$PADDLE_ROOT
/tools/check_added_ut.sh
if
[
${
PRECISION_TEST
:-
OFF
}
==
"ON"
]
;
then
precision_cases
=
`
python
$PADDLE_ROOT
/tools/get_pr_ut.py
`
python3.7
$PADDLE_ROOT
/tools/get_pr_ut.py
if
[[
-f
"ut_list"
]]
;
then
set
+x
precision_cases
=
`
cat
ut_list
`
set
-x
fi
fi
bash
$PADDLE_ROOT
/tools/check_added_ut.sh
if
[
-a
"
$PADDLE_ROOT
/added_ut"
]
;
then
added_uts
=
^
$(
awk
BEGIN
{
RS
=
EOF
}
'{gsub(/\n/,"$|^");print}'
$PADDLE_ROOT
/added_ut
)
$
ctest
-R
"(
${
added_uts
}
)"
--output-on-failure
--repeat-until-fail
3
--timeout
15
;
added_ut_error
=
$?
...
...
@@ -1060,6 +1065,7 @@ set +x
exit
8
;
fi
fi
set
+x
EXIT_CODE
=
0
;
test_cases
=
$(
ctest
-N
-V
)
# get all test cases
exclusive_tests
=
''
# cases list which would be run exclusively
...
...
tools/check_added_ut.sh
浏览文件 @
e63a68fe
...
...
@@ -14,7 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
set
-e
set
+e
set
-x
if
[
-z
${
BRANCH
}
]
;
then
BRANCH
=
"develop"
fi
...
...
@@ -25,6 +26,7 @@ CURDIR=`pwd`
cd
$PADDLE_ROOT
cp
$PADDLE_ROOT
/paddle/scripts/paddle_build.sh
$PADDLE_ROOT
/paddle/scripts/paddle_build_pre.sh
CURBRANCH
=
`
git rev-parse
--abbrev-ref
HEAD
`
echo
$CURBRANCH
git checkout
-b
prec_added_ut upstream/
${
BRANCH
}
mkdir
prec_build
cd
prec_build
...
...
@@ -32,13 +34,14 @@ bash $PADDLE_ROOT/paddle/scripts/paddle_build_pre.sh cmake_gen_in_current_dir >p
ctest
-N
|
awk
-F
':'
'{print $2}'
|
sed
'/^$/d'
|
sed
'$d'
|
sed
's/ //g'
>
/
$PADDLE_ROOT
/br-ut
cd
$PADDLE_ROOT
/build
ctest
-N
|
awk
-F
':'
'{print $2}'
|
sed
'/^$/d'
|
sed
'$d'
|
sed
's/ //g'
>
/
$PADDLE_ROOT
/pr-ut
cd
/
$PADDLE_ROOT
grep
-F
-x
-v
-f
br-ut pr-ut
>
/
$PADDLE_ROOT
/added_ut
cd
$PADDLE_ROOT
grep
-F
-x
-v
-f
br-ut pr-ut
>
$PADDLE_ROOT
/added_ut
echo
"New-UT:"
cat
/
$PADDLE_ROOT
/added_ut
cat
$PADDLE_ROOT
/added_ut
rm
-rf
prec_build
rm
/
$PADDLE_ROOT
/br-ut /
$PADDLE_ROOT
/pr-ut
$PADDLE_ROOT
/paddle/scripts/paddle_build_pre.sh
rm
$PADDLE_ROOT
/br-ut
$PADDLE_ROOT
/pr-ut
$PADDLE_ROOT
/paddle/scripts/paddle_build_pre.sh
git checkout
$CURBRANCH
echo
$CURBRANCH
git branch
-D
prec_added_ut
cd
$CURDIR
export
CI_SKIP_CPP_TEST
=
tools/coverage/paddle_coverage.sh
浏览文件 @
e63a68fe
...
...
@@ -28,7 +28,7 @@ make install
cd
/paddle/build
python3
${
PADDLE_ROOT
}
/tools/coverage/gcda_clean.py
${
GIT_PR_ID
}
python3
.7
${
PADDLE_ROOT
}
/tools/coverage/gcda_clean.py
${
GIT_PR_ID
}
lcov
--capture
-d
./
-o
coverage.info
--rc
lcov_branch_coverage
=
0
...
...
@@ -67,9 +67,9 @@ gen_full_html_report || true
function
gen_diff_html_report
()
{
if
[
"
${
GIT_PR_ID
}
"
!=
""
]
;
then
COVERAGE_DIFF_PATTERN
=
"
`
python3
${
PADDLE_ROOT
}
/tools/coverage/pull_request.py files
${
GIT_PR_ID
}
`
"
COVERAGE_DIFF_PATTERN
=
"
`
python3
.7
${
PADDLE_ROOT
}
/tools/coverage/pull_request.py files
${
GIT_PR_ID
}
`
"
python3
${
PADDLE_ROOT
}
/tools/coverage/pull_request.py diff
${
GIT_PR_ID
}
>
git-diff.out
python3
.7
${
PADDLE_ROOT
}
/tools/coverage/pull_request.py diff
${
GIT_PR_ID
}
>
git-diff.out
fi
lcov
--extract
coverage-full.info
\
...
...
@@ -77,7 +77,7 @@ function gen_diff_html_report() {
-o
coverage-diff.info
\
--rc
lcov_branch_coverage
=
0
python3
${
PADDLE_ROOT
}
/tools/coverage/coverage_diff.py coverage-diff.info git-diff.out
>
coverage-diff.tmp
python3
.7
${
PADDLE_ROOT
}
/tools/coverage/coverage_diff.py coverage-diff.info git-diff.out
>
coverage-diff.tmp
mv
-f
coverage-diff.tmp coverage-diff.info
...
...
@@ -96,7 +96,7 @@ set -x
coverage xml
-i
-o
python-coverage.xml
python3
${
PADDLE_ROOT
}
/tools/coverage/python_coverage.py
>
python-coverage.info
python3
.7
${
PADDLE_ROOT
}
/tools/coverage/python_coverage.py
>
python-coverage.info
# python full html report
#
...
...
@@ -122,9 +122,9 @@ gen_python_full_html_report || true
function
gen_python_diff_html_report
()
{
if
[
"
${
GIT_PR_ID
}
"
!=
""
]
;
then
COVERAGE_DIFF_PATTERN
=
"
`
python
${
PADDLE_ROOT
}
/tools/coverage/pull_request.py files
${
GIT_PR_ID
}
`
"
COVERAGE_DIFF_PATTERN
=
"
`
python
3.7
${
PADDLE_ROOT
}
/tools/coverage/pull_request.py files
${
GIT_PR_ID
}
`
"
python
${
PADDLE_ROOT
}
/tools/coverage/pull_request.py diff
${
GIT_PR_ID
}
>
python-git-diff.out
python
3.7
${
PADDLE_ROOT
}
/tools/coverage/pull_request.py diff
${
GIT_PR_ID
}
>
python-git-diff.out
fi
lcov
--extract
python-coverage-full.info
\
...
...
@@ -132,7 +132,7 @@ function gen_python_diff_html_report() {
-o
python-coverage-diff.info
\
--rc
lcov_branch_coverage
=
0
python
${
PADDLE_ROOT
}
/tools/coverage/coverage_diff.py python-coverage-diff.info python-git-diff.out
>
python-coverage-diff.tmp
python
3.7
${
PADDLE_ROOT
}
/tools/coverage/coverage_diff.py python-coverage-diff.info python-git-diff.out
>
python-coverage-diff.tmp
mv
-f
python-coverage-diff.tmp python-coverage-diff.info
...
...
@@ -150,11 +150,11 @@ gen_python_diff_html_report || true
echo
"Assert Diff Coverage"
python
${
PADDLE_ROOT
}
/tools/coverage/coverage_lines.py coverage-diff.info 0.9
||
COVERAGE_LINES_ASSERT
=
1
python
3.7
${
PADDLE_ROOT
}
/tools/coverage/coverage_lines.py coverage-diff.info 0.9
||
COVERAGE_LINES_ASSERT
=
1
echo
"Assert Python Diff Coverage"
python
${
PADDLE_ROOT
}
/tools/coverage/coverage_lines.py python-coverage-diff.info 0.9
||
PYTHON_COVERAGE_LINES_ASSERT
=
1
python
3.7
${
PADDLE_ROOT
}
/tools/coverage/coverage_lines.py python-coverage-diff.info 0.9
||
PYTHON_COVERAGE_LINES_ASSERT
=
1
if
[
"
$COVERAGE_LINES_ASSERT
"
=
"1"
]
||
[
"
$PYTHON_COVERAGE_LINES_ASSERT
"
=
"1"
]
;
then
echo
"exit 9"
>
/tmp/paddle_coverage.result
...
...
tools/get_pr_ut.py
浏览文件 @
e63a68fe
...
...
@@ -17,6 +17,8 @@ import os
import
json
import
re
import
sys
import
time
import
subprocess
import
requests
from
github
import
Github
...
...
@@ -45,7 +47,7 @@ class PRChecker(object):
""" Get pull request. """
pr_id
=
os
.
getenv
(
'GIT_PR_ID'
)
if
not
pr_id
:
print
(
'No PR ID'
)
print
(
'
PREC
No PR ID'
)
exit
(
0
)
suffix
=
os
.
getenv
(
'PREC_SUFFIX'
)
if
suffix
:
...
...
@@ -60,9 +62,31 @@ class PRChecker(object):
else
:
break
ix
=
ix
+
1
if
last_commit
.
message
.
find
(
'test=full_case'
)
!=
-
1
:
if
last_commit
.
message
.
find
(
'test=allcase'
)
!=
-
1
:
print
(
'PREC test=allcase is set'
)
self
.
full_case
=
True
#todo: exception
def
__wget_with_retry
(
self
,
url
):
ix
=
1
proxy
=
'--no-proxy'
while
ix
<
6
:
if
ix
//
2
==
0
:
proxy
=
''
else
:
proxy
=
'--no-proxy'
code
=
subprocess
.
call
(
'wget -q {} --no-check-certificate {}'
.
format
(
proxy
,
url
),
shell
=
True
)
if
code
==
0
:
return
True
print
(
'PREC download {} error, retry {} time(s) after {} secs.[proxy_option={}]'
.
format
(
url
,
ix
,
ix
*
10
,
proxy
))
time
.
sleep
(
ix
*
10
)
ix
+=
1
return
False
def
get_pr_files
(
self
):
""" Get files in pull request. """
page
=
0
...
...
@@ -101,6 +125,7 @@ class PRChecker(object):
def
get_comment_of_file
(
self
,
f
):
#content = self.repo.get_contents(f.replace(PADDLE_ROOT, ''), 'pull/').decoded_content
#todo: get file from github
with
open
(
f
)
as
fd
:
lines
=
fd
.
readlines
()
lineno
=
1
...
...
@@ -166,6 +191,7 @@ class PRChecker(object):
for
l
in
diff_lines
:
if
l
not
in
comment_lines
:
return
False
print
(
'PREC {} is only comment'
.
format
(
f
))
return
True
def
get_pr_ut
(
self
):
...
...
@@ -175,8 +201,12 @@ class PRChecker(object):
check_added_ut
=
False
ut_list
=
[]
file_ut_map
=
None
cmd
=
'wget -q --no-proxy --no-check-certificate https://sys-p0.bj.bcebos.com/prec/file_ut.json'
+
self
.
suffix
os
.
system
(
cmd
)
ret
=
self
.
__wget_with_retry
(
'https://sys-p0.bj.bcebos.com/prec/file_ut.json{}'
.
format
(
self
.
suffix
))
if
not
ret
:
print
(
'PREC download file_ut.json failed'
)
exit
(
1
)
with
open
(
'file_ut.json'
+
self
.
suffix
)
as
jsonfile
:
file_ut_map
=
json
.
load
(
jsonfile
)
for
f
in
self
.
get_pr_files
():
...
...
@@ -187,16 +217,24 @@ class PRChecker(object):
if
self
.
is_only_comment
(
f
):
ut_list
.
append
(
'h_cu_comment_placeholder'
)
else
:
print
(
'PREC dismatch: {} not in file ut map and not md or comment'
.
format
(
f
))
return
''
elif
f
.
endswith
(
'.cc'
)
or
f
.
endswith
(
'.py'
)
or
f
.
endswith
(
'.cu'
):
if
f
.
find
(
'test_'
)
!=
-
1
or
f
.
find
(
'_test'
)
!=
-
1
:
print
(
'PREC {} need check new ut'
.
format
(
f
))
check_added_ut
=
True
elif
self
.
is_only_comment
(
f
):
ut_list
.
append
(
'nomap_comment_placeholder'
)
else
:
print
(
'PREC dismatch: {} not in file ut map and not new ut or comment'
.
format
(
f
))
return
''
else
:
print
(
'PREC dismatch: {} not in file ut map'
.
format
(
f
))
return
''
else
:
if
self
.
is_only_comment
(
f
):
...
...
@@ -204,24 +242,28 @@ class PRChecker(object):
else
:
ut_list
.
extend
(
file_ut_map
.
get
(
f
))
ut_list
=
list
(
set
(
ut_list
))
cmd
=
'wget -q --no-proxy --no-check-certificate https://sys-p0.bj.bcebos.com/prec/prec_delta'
+
self
.
suffix
os
.
system
(
cmd
)
with
open
(
'prec_delta'
+
self
.
suffix
)
as
delta
:
for
ut
in
delta
:
ut_list
.
append
(
ut
.
rstrip
(
'
\r\n
'
))
ret
=
self
.
__wget_with_retry
(
'https://sys-p0.bj.bcebos.com/prec/prec_delta{}'
.
format
(
self
.
suffix
))
if
ret
:
with
open
(
'prec_delta'
+
self
.
suffix
)
as
delta
:
for
ut
in
delta
:
ut_list
.
append
(
ut
.
rstrip
(
'
\r\n
'
))
else
:
print
(
'PREC download prec_delta failed'
)
if
check_added_ut
:
cmd
=
'bash {}/tools/check_added_ut.sh >/tmp/pre_ut 2>&1'
.
format
(
PADDLE_ROOT
)
os
.
system
(
cmd
)
with
open
(
'{}/added_ut'
.
format
(
PADDLE_ROOT
))
as
utfile
:
for
ut
in
utfile
:
print
(
'PREC NEW UT: {}'
.
format
(
ut
.
rstrip
(
'
\r\n
'
)))
ut_list
.
append
(
ut
.
rstrip
(
'
\r\n
'
))
return
'
'
.
join
(
ut_list
)
return
'
\n
'
.
join
(
ut_list
)
if
__name__
==
'__main__'
:
pr_checker
=
PRChecker
()
pr_checker
.
init
()
print
(
pr_checker
.
get_pr_ut
())
#print(pr_checker.get_pr_ut())
with
open
(
'ut_list'
,
'w'
)
as
f
:
f
.
write
(
pr_checker
.
get_pr_ut
())
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录