Skip to content

P
Paddle

吃玉米的猫 / Paddle
与 Fork 源项目一致

Fork自 PaddlePaddle / Paddle

通知 1
Star 0
Fork 0
P
Paddle

体验新版 GitCode，发现更多精彩内容 >>

提交 98f1cebd 编写于 作者: Y Yucheng 提交者： Dong Daxiang
浏览文件
操作

add sample code test under python3 and enabled multi-thread (#20950) 

* test=develop
上级 394edd86
隐藏空白更改
内联 并排
Showing with 209 addition and 495 deletion
paddle/scripts/paddle_build.sh
浏览文件 @ 98f1cebd
......@@ -1002,8 +1002,7 @@ function build_document_preview() {
function example() {
pip install ${PADDLE_ROOT}/build/python/dist/*.whl
paddle version
cp ${PADDLE_ROOT}/tools/sampcd_processor.py ${PADDLE_ROOT}/python/paddle/fluid
cd ${PADDLE_ROOT}/python/paddle/fluid
cd ${PADDLE_ROOT}/tools
python sampcd_processor.py cpu
if [ "$?" != "0" ];then
echo "Code instance execution failed"
......
tools/sampcd_processor.py
浏览文件 @ 98f1cebd
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
......@@ -15,53 +15,60 @@
import os
import sys
import subprocess
import multiprocessing
import math
import platform
"""
please make sure to run in the tools path
usage: python sample_test.py {arg1} {arg2}
arg1: the first arg defined running in gpu version or cpu version
arg2: the second arg defined testing python2 or python3
for example, you can run cpu version python2 testing like this:
python sample_test.py cpu 2
"""
def find_all(srcstr, substr):
'''
"""
to find all desired substring in the source string
and return their starting indices as a list
Args:
srcstr(str): the parent string
substr(str): substr
Returns:
list: a list of the indices of the substrings
list: a list of the indices of the substrings
found
'''
"""
indices = []
gotone = srcstr.find(substr)
while (gotone != -1):
indices.append(gotone)
gotone = srcstr.find(substr, gotone + 1)
return indices
def check_indent(cdline):
'''
"""
to check the indent of a given code line
to get the number of starting blank chars,
e.t. blankspaces and \t
\t will be interpreted as 4 single blankspaces,
\t will be interpreted as 4 single blankspaces,
e.t. '\t'=' '
Args:
cdline(str) : a single line of code from the source file
Returns:
int : the indent of the number of interpreted
int : the indent of the number of interpreted
blankspaces
'''
"""
indent = 0
for c in cdline:
if c == '\t':
......@@ -70,24 +77,16 @@ def check_indent(cdline):
indent += 1
if c != ' ' and c != '\t':
break
return indent
#srccom: raw comments in the source,including ''' and original indent
def sampcd_extract_and_run(srccom,
name,
logf,
htype="def",
hname="",
show_details=False):
'''
# srccom: raw comments in the source,including ''' and original indent
def sampcd_extract_and_run(srccom, name, htype="def", hname=""):
"""
Extract and run sample codes from source comment and
the result will be returned.
As an ultimate result, this function returns a list of
As an ultimate result, this function returns a list of
status codes for each sample code (in top-down order)
found in srccom.
......@@ -97,12 +96,12 @@ def sampcd_extract_and_run(srccom,
2:have sample code but format is wrong
1:no sample code
0:successful
-1:no comments found
-1:no comments found
-2:in white list
there may be several examples in a source comment
so status deserves a list to contain the states.
For instance, some API has three example codes,
For instance, some API has three example codes,
code 1 is successful, code 2 is error, code 3 is successful
so the list to return is [0,3,0]
......@@ -110,111 +109,79 @@ def sampcd_extract_and_run(srccom,
srccom(str): the source comment of some API whose
example codes will be extracted and run.
name(str): the name of the API.
logf(file): for logging the output in case they are
flushed.
htype(str): the type of hint banners, def/class/method.
hname(str): the name of the hint banners , e.t. def hname.
show_details(bool): Set it to False to print wrong sample
codes only.
Returns:
list: the status code of all the sample codes found in srccom.
'''
def sampcd_header_print(name, sampcd, htype, hname, logf):
'''
"""
def sampcd_header_print(name, sampcd, htype, hname):
"""
print hint banner headers.
Args:
name(str): the name of the API.
sampcd(str): sample code string
htype(str): the type of hint banners, def/class/method.
hname(str): the name of the hint banners , e.t. def hname.
logf(file): for logging the output in case they are
flushed.
'''
print_header(logf, htype, hname)
print "Sample code " + str(y) + " extracted for " + name + " :"
print "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"
"""
print_header(htype, hname)
print("Sample code ", str(y), " extracted for ", name, " :")
print(sampcd)
print "----example code check----\n"
print "executing sample code ....."
print "execution result:"
logf.write("\nSample code extracted for " + name + " :\n")
logf.write("\n" + sampcd + "\n")
logf.write("\n----example code check----\n")
logf.write("\nexecuting sample code .....\n")
logf.write("\nexecution result:\n")
print("----example code check----\n")
print("executing sample code .....")
print("execution result:")
sampcd_begins = find_all(srccom, " code-block:: python")
status = []
if (len(sampcd_begins) == 0):
print_header(logf, htype, hname)
if len(sampcd_begins) == 0:
print_header(htype, hname)
'''
detect sample codes using >>> to format
and consider this situation as wrong
'''
if (srccom.find("Examples:") != -1):
print "----example code check----\n"
logf.write("\n----example code check----\n")
if (srccom.find(">>>") != -1):
logf.write(
"Deprecated sample code style:\n\n Examples:\n\n >>>codeline\n >>>codeline\n\n\n "
+ "Please use '.. code-block:: python' to " +
"format sample code.\n")
if srccom.find("Examples:") != -1:
print("----example code check----\n")
if srccom.find(">>>") != -1:
print(
"Deprecated sample code style:\n\n Examples:\n\n >>>codeline\n >>>codeline\n\n\n "
+ "Please use '.. code-block:: python' to " +
"Deprecated sample code style:\n\n Examples:\n\n >>>codeline\n >>>codeline\n\n\n ",
"Please use '.. code-block:: python' to ",
"format sample code.\n")
status.append(2)
print "status code for all sample codes in " + name + " : " + str(
status)
print("status code for all sample codes in ", name, " : ",
str(status))
else:
print "No sample code!\n"
logf.write("\nNo sample code!\n")
print("No sample code!\n")
status.append(1)
print "status code for all sample codes in " + name + " : " + str(
status)
print("status code for all sample codes in ", name, " : ",
str(status))
for y in range(1, len(sampcd_begins) + 1):
sampcd_begin = sampcd_begins[y - 1]
sampcd = srccom[sampcd_begin + len(" code-block:: python") + 1:]
sampcd = sampcd.split("\n")
#remove starting empty lines
# remove starting empty lines
while sampcd[0].replace(' ', '').replace('\t', '') == '':
sampcd.pop(0)
#the mininmum indent, which is the indent of the first
#non-empty line
# the minimum indent, which is the indent of the first
# non-empty line
min_indent = check_indent(sampcd[0])
sampcd_to_write = []
for i in range(0, len(sampcd)):
cdline = sampcd[i]
#handle empty lines or those only with spaces/tabs
# handle empty lines or those only with spaces/tabs
if cdline.strip() == '':
continue
this_indent = check_indent(cdline)
if (this_indent < min_indent):
if this_indent < min_indent:
break
else:
cdline = cdline.replace('\t', ' ')
sampcd_to_write.append(cdline[min_indent:])
......@@ -224,94 +191,75 @@ def sampcd_extract_and_run(srccom,
sampcd = '\nimport os\n' + 'os.environ["CUDA_VISIBLE_DEVICES"] = ""\n' + sampcd
if sys.argv[1] == "gpu":
sampcd = '\nimport os\n' + 'os.environ["CUDA_VISIBLE_DEVICES"] = "0"\n' + sampcd
sampcd += '\nprint ' + '\"' + name + ' sample code is executed successfully!\"\n'
sampcd += '\nprint(' + '\"' + name + ' sample code is executed successfully!\")'
if (len(sampcd_begins) > 1):
if len(sampcd_begins) > 1:
tfname = name + "_example_" + str(y) + ".py"
else:
tfname = name + "_example" + ".py"
tempf = open("samplecode_temp/" + tfname, 'w')
tempf.write(sampcd)
tempf.close()
cmd = ["python", "samplecode_temp/" + tfname]
if platform.python_version()[0] == "2":
cmd = ["python", "samplecode_temp/" + tfname]
elif platform.python_version()[0] == "3":
cmd = ["python3", "samplecode_temp/" + tfname]
else:
print("fail to parse python version!")
exit(1)
subprc = subprocess.Popen(
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output, error = subprc.communicate()
msg = "".join(output)
err = "".join(error)
if (subprc.returncode != 0):
print("\nSample code error found in " + name + ":\n")
sampcd_header_print(name, sampcd, htype, hname, logf)
print "subprocess return code: " + str(subprc.returncode)
print("Error Raised from Sample Code " + name + " :\n")
print err
print msg
logf.write("\nError Raised from Sample Code " + name + " :\n")
logf.write("\n" + msg + "\n")
msg = "".join(output.decode(encoding='utf-8'))
err = "".join(error.decode(encoding='utf-8'))
if subprc.returncode != 0:
print("\nSample code error found in ", name, ":\n")
sampcd_header_print(name, sampcd, htype, hname)
print("subprocess return code: ", str(subprc.returncode))
print("Error Raised from Sample Code ", name, " :\n")
print(err)
print(msg)
status.append(3)
print "status code for all sample codes in " + name + str(status)
#It works!
print("status code for all sample codes in ", name, str(status))
# It works!
else:
status.append(0)
if show_details:
sampcd_header_print(name, sampcd, htype, hname, logf)
print "subprocess return code: " + str(subprc.returncode)
print msg
logf.write("\n" + msg + "\n")
print "status code for all sample codes in " + name + " : " + str(
status)
#msg is the returned code execution report
# msg is the returned code execution report
os.remove("samplecode_temp/" + tfname)
return status
def single_defcom_extract(start_from, srcls, is_class_begin=False):
'''
"""
to extract a def function/class/method comments body
Args:
Args:
start_from(int): the line num of "def" header
srcls(list): the source file in lines
is_class_begin(bool): whether the start_from is a beginning a class. \
For a sole class body itself may end up with its method if it has no
docstring. But the body of \
a common def function can only be ended up by a none-indented def/class
Returns:
string : the extracted comment body, inclusive of its quote marks.
'''
"""
i = start_from
fcombody = "" #def comment body
comstart = -1 # the starting line index of comment mark "'''" or """"""
#if it is not -1, it indicates the loop is in the comment body
fcombody = "" # def comment body
comstart = -1 # the starting line index of comment mark "'''" or """"""
# if it is not -1, it indicates the loop is in the comment body
comstyle = 0 # comment mark style ,comments quoted with ''' is coded as 1
# comments quoted with """ is coded as 2
for x in range(i + 1, len(srcls)):
if is_class_begin:
if (srcls[x].replace('\t', ' ').startswith(' def ')):
if srcls[x].replace('\t', ' ').startswith(' def '):
break
if ((srcls[x].startswith('def ') or srcls[x].startswith('class '))):
if srcls[x].startswith('def ') or srcls[x].startswith('class '):
break
else:
if (comstart == -1 and srcls[x].replace(" ", '').replace(
"\t", '').replace("\n", '').startswith("\"\"\"")):
comstart = x
......@@ -321,7 +269,6 @@ def single_defcom_extract(start_from, srcls, is_class_begin=False):
srcls[x].replace(" ", '').replace("\t", '').replace(
"\n", '').startswith("\"\"\"")):
break
if (comstart == -1 and srcls[x].replace(" ", '').replace(
"\t", '').replace("\n", '').startswith("\'\'\'")):
comstart = x
......@@ -332,99 +279,61 @@ def single_defcom_extract(start_from, srcls, is_class_begin=False):
"\n", '').startswith("\'\'\'")):
break
if (comstart !=
-1): #when the comments start, begin to add line to fcombody
-1): # when the comments start, begin to add line to fcombody
fcombody += srcls[x]
return fcombody
def print_header(logf, htype, name):
print htype + " name:" + name
print "-----------------------"
logf.write("\n\n" + htype + " name:" + name + "\n")
logf.write("-----------------------\n")
def srcf_print(srcfile):
def print_header(htype, name):
print(htype, " name:", name)
print("-----------------------")
print "source file name:" + srcfile.name
print "---------------------------------------------------"
logf.write("source file name:" + srcfile.name + "\n")
logf.write("---------------------------------------------------\n\n")
def show_alllist(alllist):
print "__all__:" + str(alllist) + "\n"
logf.write("__all__:" + str(alllist) + "\n\n")
def srccoms_extract(srcfile, logf, status_all, wlist, show_details):
'''
def srccoms_extract(srcfile, status_all, wlist):
"""
Given a source file ``srcfile``, this function will
extract its API(doc comments) and run sample codes in the
API.
Args:
srcfile(file): the source file
logf(file): log recording file
status_all(dict): record all the sample code execution states.
wlist(list): white list
show_details(bool): if show_details is True, the whole process will be printed for you
to debug it locally
Returns:
string: the length of __all__ list in srcfile versus the exact number of
analysed API to make sure no API is missed in this srcfile and it
is useful for statistic practices.
'''
"""
srcc = srcfile.read()
#2. get defs and classes header line number
#set file pointer to its beginning
# 2. get defs and classes header line number
# set file pointer to its beginning
srcfile.seek(0, 0)
srcls = srcfile.readlines() #source lines
if show_details:
srcf_print(srcfile)
srcls = srcfile.readlines() # source lines
#1. fetch__all__ list
# 1. fetch__all__ list
allidx = srcc.find("__all__")
if (allidx != -1):
if allidx != -1:
alllist = []
#get all list for layers/ops.py
if (srcfile.name.find("ops.py") != -1):
# get all list for layers/ops.py
if srcfile.name.find("ops.py") != -1:
for ai in range(0, len(srcls)):
if (srcls[ai].startswith("__all__")):
if srcls[ai].startswith("__all__"):
lb = srcls[ai].find('[')
rb = srcls[ai].find(']')
if (lb == -1):
if lb == -1:
continue
allele = srcls[ai][lb + 1:rb].replace("'", '').replace(
" ", '').replace("\"", '')
alllist.append(allele)
if '' in alllist:
alllist.remove('')
if show_details:
show_alllist(alllist)
else:
alllist_b = allidx + len("__all__")
allstr = srcc[alllist_b + srcc[alllist_b:].find("[") + 1:alllist_b +
srcc[alllist_b:].find("]")]
allstr = allstr.replace("\n", '').replace(" ", '').replace(
......@@ -432,295 +341,184 @@ def srccoms_extract(srcfile, logf, status_all, wlist, show_details):
alllist = allstr.split(',')
if '' in alllist:
alllist.remove('')
if show_details:
show_alllist(alllist)
api_alllist_count = len(alllist)
api_count = 0
handled = []
#get src contents in layers/ops.py
if (srcfile.name.find("ops.py") != -1):
# get src contents in layers/ops.py
if srcfile.name.find("ops.py") != -1:
for i in range(0, len(srcls)):
if srcls[i].find("__doc__") != -1:
opname = srcls[i][:srcls[i].find("__doc__") - 1]
if opname in wlist:
status_all[srcfile.name + '/' + opname] = [-2]
if show_details:
print_header(logf, "def", opname)
print opname + " is in white list, thus skipped"
logf.write("\n" + opname +
" is in white list, thus skipped\n")
print status_all[srcfile.name + '/' + opname]
logf.write("\n" + "execution status" + str(
status_all[srcfile.name + '/' + opname]) + "\n")
continue
comstart = i
for j in range(i, len(srcls)):
if (srcls[j].find("\"\"\"") != -1):
if srcls[j].find("\"\"\"") != -1:
comstart = i
opcom = ""
for j in range(comstart + 1, len(srcls)):
opcom += srcls[j]
if (srcls[j].find("\"\"\"") != -1):
if srcls[j].find("\"\"\"") != -1:
break
status = sampcd_extract_and_run(opcom, opname, logf, "def",
opname, show_details)
status = sampcd_extract_and_run(opcom, opname, "def",
opname)
api_count += 1
status_all[srcfile.name + '/' + opname] = status
handled.append(
opname) #ops.py also has normal formatted functions
#use list 'handled' to mark the functions have been handled here
#which will be ignored in the following step
opname) # ops.py also has normal formatted functions
# use list 'handled' to mark the functions have been handled here
# which will be ignored in the following step
for i in range(0, len(srcls)):
if srcls[i].startswith(
'def '): #a function header is detected in line i
'def '): # a function header is detected in line i
f_header = srcls[i].replace(" ", '')
fn = f_header[len('def'):f_header.find('(')] #function name
fn = f_header[len('def'):f_header.find('(')] # function name
if fn in handled:
continue
if fn in alllist:
api_count += 1
if fn in wlist or fn + "@" + srcfile.name in wlist:
status_all[srcfile.name + '/' + fn] = [-2]
if show_details:
print_header(logf, "def", fn)
print fn + " is in white list, thus skipped"
logf.write("\n" + fn +
" is in white list, thus skipped\n")
print status_all[srcfile.name + '/' + fn]
logf.write("\n" + "execution status" + str(
status_all[srcfile.name + '/' + fn]) + "\n")
continue
fcombody = single_defcom_extract(i, srcls)
if (fcombody == ""): #if no comment
print_header(logf, "def", fn)
print "WARNING: no comments in function " + fn + ", but it deserves."
logf.write("no comments in function " + fn + "\n\n")
if fcombody == "": # if no comment
print_header("def", fn)
print("WARNING: no comments in function ", fn,
", but it deserves.")
status_all[srcfile.name + '/' + fn] = [-1]
print status_all[srcfile.name + '/' + fn]
logf.write("\n" + "execution status" + str(status_all[
srcfile.name + '/' + fn]) + "\n")
print(status_all[srcfile.name + '/' + fn])
continue
else:
status = sampcd_extract_and_run(fcombody, fn, logf,
"def", fn, show_details)
status = sampcd_extract_and_run(fcombody, fn, "def", fn)
status_all[srcfile.name + '/' + fn] = status
else:
if show_details:
print_header(logf, "def", fn)
print fn + " not in __all__ list"
logf.write(fn + " not in __all__ list\n\n")
if srcls[i].startswith('class '):
c_header = srcls[i].replace(" ", '')
cn = c_header[len('class'):c_header.find('(')] #class name
cn = c_header[len('class'):c_header.find('(')] # class name
if cn in handled:
continue
if cn in alllist:
api_count += 1
if cn in wlist or cn + "@" + srcfile.name in wlist:
status_all[srcfile.name + '/' + cn] = [-2]
if show_details:
print cn + " is in white list, thus skipped"
logf.write("\n" + cn +
" is in white list, thus skipped\n")
print status_all[srcfile.name + '/' + cn]
logf.write("\n" + "execution status" + str(
status_all[srcfile.name + '/' + cn]) + "\n")
continue
#class comment
# class comment
classcom = single_defcom_extract(i, srcls, True)
if (classcom != ""):
status = sampcd_extract_and_run(
classcom, cn, logf, "class", cn, show_details)
if classcom != "":
status = sampcd_extract_and_run(classcom, cn, "class",
cn)
status_all[srcfile.name + '/' + cn] = status
else:
print "WARNING: no comments in class itself " + cn + ", but it deserves.\n"
logf.write("no comments in class itself " + cn +
"\n\n\n")
print("WARNING: no comments in class itself ", cn,
", but it deserves.\n")
status_all[srcfile.name + '/' + cn] = [-1]
print status_all[srcfile.name + '/' + cn]
logf.write("\n" + "execution status" + str(status_all[
srcfile.name + '/' + cn]) + "\n")
#handling methods in class bodies
print(status_all[srcfile.name + '/' + cn])
# handling methods in class bodies
for x in range(
i + 1,
len(srcls)): #from the next line of class header
len(srcls)): # from the next line of class header
if (srcls[x].startswith('def ') or
srcls[x].startswith('class ')):
break
else:
#member method def header
# member method def header
srcls[x] = srcls[x].replace('\t', ' ')
if (srcls[x].startswith(
' def ')): #detect a mehtod header..
' def ')): # detect a mehtod header..
thisl = srcls[x]
indent = len(thisl) - len(thisl.lstrip())
mn = thisl[indent + len('def '):thisl.find(
'(')] #method name
name = cn + "." + mn #full name
'(')] # method name
name = cn + "." + mn # full name
if mn.startswith('_'):
if show_details:
print mn + " is hidden, not visible to users\n"
logf.write(
"\n" + mn +
" is hidden, not visible to users\n")
continue
if name in wlist or name + "@" + srcfile.name in wlist:
status_all[srcfile.name + '/' + name] = [-2]
if show_details:
print name + " is in white list, thus skipped"
logf.write(
"\n" + name +
" is in white list, thus skipped\n")
print status_all[srcfile.name + '/' +
name]
logf.write(
"\n" + "execution status" + str(
status_all[srcfile.name + '/' +
name]) + "\n")
continue
thismethod = [] #method body lines
thismethod.append(thisl[indent:])
#get all the lines of a single method body
#into thismethod(list)
#and send it to single_defcom_extract
thismethod = [thisl[indent:]
] # method body lines
# get all the lines of a single method body
# into thismethod(list)
# and send it to single_defcom_extract
for y in range(x + 1, len(srcls)):
srcls[y] = srcls[y].replace('\t', ' ')
if (srcls[y].startswith('def ') or
srcls[y].startswith('class ')):
#end of method
# end of method
break
elif (srcls[y].startswith(' def ')):
#end of method
elif srcls[y].startswith(' def '):
# end of method
break
else:
thismethod.append(srcls[y][indent:])
thismtdcom = single_defcom_extract(0,
thismethod)
if (thismtdcom != ""):
if thismtdcom != "":
status = sampcd_extract_and_run(
thismtdcom, name, logf, "method", name,
show_details)
thismtdcom, name, "method", name)
status_all[srcfile.name + '/' +
name] = status
else:
if show_details:
print "no comments in method " + name + "\n"
logf.write("no comments in method " +
name + "\n\n\n")
status_all[srcfile.name + '/' +
name] = [-1]
print status_all[srcfile.name + '/' +
name]
logf.write(
"\n" + "execution status" + str(
status_all[srcfile.name + '/' +
name]) + "\n")
else:
if show_details:
print cn + " is not in __all__ list"
logf.write(cn + " is not in __all__ list\n\n")
return [
srcfile.name + " all list length: " + str(api_alllist_count),
"analysed api count: " + str(api_count)
]
def test(file_list):
for file in file_list:
src = open(file, 'r')
counts = srccoms_extract(src, status_all, wlist)
src.close()
'''
Important constant lists:
filenames : the modules pending for check .
wlist : a list of API that should not trigger the example check .
It is composed of wlist_temp + wlist_inneed + wlist_ignore.
show_details: a boolean value to indicate whether it should be run
in debugging mode.
status_all: a status list containing all the execution status of all
APIs
srcfile: the source .py code file
'''
filenames = [
"layers/control_flow.py", "layers/io.py", "layers/nn.py", "layers/ops.py",
"layers/tensor.py", "layers/learning_rate_scheduler.py",
"layers/detection.py", "layers/metric_op.py"
"../python/paddle/fluid/layers/control_flow.py",
"../python/paddle/fluid/layers/io.py",
"../python/paddle/fluid/layers/nn.py",
"../python/paddle/fluid/layers/ops.py",
"../python/paddle/fluid/layers/tensor.py",
"../python/paddle/fluid/layers/learning_rate_scheduler.py",
"../python/paddle/fluid/layers/detection.py",
"../python/paddle/fluid/layers/metric_op.py"
]
filenames += [
"dygraph/layers.py", "dygraph/base.py", "dygraph/nn.py",
"dygraph/tracer.py", "dygraph/profiler.py", "dygraph/parallel.py",
"dygraph/checkpoint.py", "dygraph/learning_rate_scheduler.py",
"dygraph/backward_strategy.py"
"../python/paddle/fluid/dygraph/layers.py",
"../python/paddle/fluid/dygraph/base.py",
"../python/paddle/fluid/dygraph/nn.py",
"../python/paddle/fluid/dygraph/tracer.py",
"../python/paddle/fluid/dygraph/profiler.py",
"../python/paddle/fluid/dygraph/parallel.py",
"../python/paddle/fluid/dygraph/checkpoint.py",
"../python/paddle/fluid/dygraph/learning_rate_scheduler.py",
"../python/paddle/fluid/dygraph/backward_strategy.py"
]
filenames += [
"data_feeder.py", "dataset.py", "clip.py", "metrics.py", "executor.py",
"initializer.py", "io.py", "nets.py", "optimizer.py", "profiler.py",
"regularizer.py", "backward.py", "average.py", "unique_name.py",
"framework.py", "evaluator.py", "param_attr.py"
"../python/paddle/fluid/data_feeder.py",
"../python/paddle/fluid/dataset.py", "../python/paddle/fluid/clip.py",
"../python/paddle/fluid/metrics.py", "../python/paddle/fluid/executor.py",
"../python/paddle/fluid/initializer.py", "../python/paddle/fluid/io.py",
"../python/paddle/fluid/nets.py", "../python/paddle/fluid/optimizer.py",
"../python/paddle/fluid/profiler.py",
"../python/paddle/fluid/regularizer.py",
"../python/paddle/fluid/backward.py", "../python/paddle/fluid/average.py",
"../python/paddle/fluid/unique_name.py",
"../python/paddle/fluid/framework.py",
"../python/paddle/fluid/evaluator.py",
"../python/paddle/fluid/param_attr.py"
]
wlist_inneed = [
"append_LARS", "BuildStrategy.debug_graphviz_path",
"BuildStrategy.enable_sequential_execution",
......@@ -752,7 +550,6 @@ wlist_inneed = [
'StaticRNN.output', "cuda_places", "CUDAPinnedPlace", "CUDAPlace",
"Program.parse_from_string"
]
wlist_temp = [
'ChunkEvaluator',
'EditDistance',
......@@ -851,7 +648,6 @@ wlist_ignore = [
'Precision.update', 'WeightedAverage.eval', 'Conv3D.forward',
'Embedding.forward', 'Recall.eval', 'FC.forward', 'While.block'
]
# only white on CPU
gpu_not_white = [
"deformable_conv", "cuda_places", "CUDAPinnedPlace", "CUDAPlace",
......@@ -861,146 +657,65 @@ gpu_not_white = [
wlist = wlist_temp + wlist_inneed + wlist_ignore
if len(sys.argv) < 2:
print "Error: inadequate number of arguments"
print("Error: inadequate number of arguments")
print('''If you are going to run it on
"CPU: >>> python sampcd_processor.py cpu
"GPU: >>> python sampcd_processor.py gpu
''')
sys.exit("lack arguments")
else:
show_details = False
if sys.argv[1] == "gpu":
for _gnw in gpu_not_white:
wlist.remove(_gnw)
elif sys.argv[1] != "cpu":
print("Unrecognized argument:'" + sys.argv[1] + "' , 'cpu' or 'gpu' is "
+ "desired\n")
print("Unrecognized argument:'", sys.argv[1], "' , 'cpu' or 'gpu' is ",
"desired\n")
sys.exit("Invalid arguments")
if len(sys.argv) == 3:
if sys.argv[2] == "sd":
show_details = True
else:
print("Unrecognized argument:'" + sys.argv[2] + "' , 'sd' is " +
"desired\n")
sys.exit("Invalid arguments")
print("* * * * * * * * * * * * * * * * * * * * * * * *\n" +
"* *\n" +
"* API check -- Example Code Cheker *\n" +
"* *\n" +
"* *\n" +
"* This process is meant to check *\n" +
"* all example codes per CI to ensure *\n" +
"* the example codes can be run successfully *\n" +
"* *\n" +
"* *\n" +
"* Refer to the comments for detailed *\n" +
"* introduction *\n" +
"* *\n" +
"* *\n" +
"* * * * * * * * * * * * * * * * * * * * * * * *\n")
print("API check -- Example Code")
print("sample_test running under python", platform.python_version())
status_all = {}
#a file to record the terminal output
logf = open("example-code-check-log.txt", 'w')
# a temp directory to store temporary sample code file
# subprocess needs a single file to run the code
# subprocess needs a single file to run the code
if not os.path.isdir("./samplecode_temp"):
os.mkdir("./samplecode_temp")
to_check = filenames
for filename in to_check:
srcfile = open(filename, 'r')
counts = srccoms_extract(srcfile, logf, status_all, wlist, show_details)
if show_details:
logf.write("\n\n" + str(counts) + "\n\n")
srcfile.close()
one_part_filenum = int(math.ceil(len(filenames) / 10))
divided_file_list = [
filenames[i:i + one_part_filenum]
for i in range(0, len(filenames), one_part_filenum)
]
po = multiprocessing.Pool(10)
for file_list in divided_file_list:
po.apply_async(test, (file_list, ))
po.close()
po.join()
# clear temp files
for root, dirs, files in os.walk("./samplecode_temp"):
for fntemp in files:
os.remove("./samplecode_temp/" + fntemp)
os.rmdir("./samplecode_temp")
status_groups = {-2: [], -1: [], 0: [], 1: [], 2: [], 3: []}
ci_pass = True
for key in status_all:
statusl = status_all[key]
for ele in statusl:
if (ele != 0 and ele != -2 and ele != -1):
if ele != 0 and ele != -2 and ele != -1:
ci_pass = False
break
if len(statusl) == 1:
status_groups[statusl[0]].append(key)
else:
for u in range(0, len(statusl)):
status_groups[statusl[u]].append(key + '_' + str(u + 1))
logf.close()
print(
"\n\n------------------End of the Check-------------------------------------------\n\n"
)
print("----------------End of the Check--------------------")
errorapisl = status_groups[1] + status_groups[2] + status_groups[3]
if len(errorapisl) > 0:
print "Error raised from: " + str(errorapisl)
print("Error raised from: ", str(errorapisl))
if not ci_pass:
print(
"\nOh no.. Mistakes found in sample codes, refer to the log for details\n\n"
)
print('''
- How to run it locally?
Simply put this script under directory:
Paddle/python/paddle/fluid/
and run in python 2.7 (as some interfaces of subprocess may
not work in python 3)
You must specify the device type to run the sample code on:
CPU: >>> python sampcd_processor.py cpu
GPU: >>> python sampcd_processor.py gpu
- How to debug?
This script has an option for showing the details of
the execution status:
>>> python sampcd_processor.py cpu sd
- NOTE:
Please ensure your are using
.. code-block:: python
[sample code starts here]
ONLY 1 BLANKSPACE between '::' and 'python'
''')
print("Mistakes found in sample codes")
exit(1)
else:
print "Sample code check is successful!"
print("Sample code check is successful!")
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或