diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh
index 06ea9d33f2cd7e650c918935725223ce4763f056..2de9d08ba5ce2bd2bfbedcd992702fbabcba9c1a 100755
--- a/paddle/scripts/paddle_build.sh
+++ b/paddle/scripts/paddle_build.sh
@@ -1002,8 +1002,7 @@ function build_document_preview() {
function example() {
pip install ${PADDLE_ROOT}/build/python/dist/*.whl
paddle version
- cp ${PADDLE_ROOT}/tools/sampcd_processor.py ${PADDLE_ROOT}/python/paddle/fluid
- cd ${PADDLE_ROOT}/python/paddle/fluid
+ cd ${PADDLE_ROOT}/tools
python sampcd_processor.py cpu
if [ "$?" != "0" ];then
echo "Code instance execution failed"
diff --git a/tools/sampcd_processor.py b/tools/sampcd_processor.py
index 6597b9378236f1b56ba7df32b0d7ba360fdb6a1b..38858b4bd39fbaf5d308b46f4b015444b7f336e9 100644
--- a/tools/sampcd_processor.py
+++ b/tools/sampcd_processor.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -15,53 +15,60 @@
import os
import sys
import subprocess
+import multiprocessing
+import math
+import platform
+"""
+please make sure to run in the tools path
+usage: python sample_test.py {arg1} {arg2}
+arg1: the first arg defined running in gpu version or cpu version
+arg2: the second arg defined testing python2 or python3
+
+for example, you can run cpu version python2 testing like this:
+
+ python sample_test.py cpu 2
+
+"""
def find_all(srcstr, substr):
- '''
+ """
to find all desired substring in the source string
and return their starting indices as a list
Args:
srcstr(str): the parent string
substr(str): substr
-
+
Returns:
- list: a list of the indices of the substrings
+ list: a list of the indices of the substrings
found
- '''
-
+ """
indices = []
-
gotone = srcstr.find(substr)
-
while (gotone != -1):
-
indices.append(gotone)
-
gotone = srcstr.find(substr, gotone + 1)
-
return indices
def check_indent(cdline):
- '''
+ """
to check the indent of a given code line
-
+
to get the number of starting blank chars,
e.t. blankspaces and \t
-
- \t will be interpreted as 4 single blankspaces,
+
+ \t will be interpreted as 4 single blankspaces,
e.t. '\t'=' '
-
+
Args:
cdline(str) : a single line of code from the source file
Returns:
- int : the indent of the number of interpreted
+ int : the indent of the number of interpreted
blankspaces
- '''
-
+ """
indent = 0
for c in cdline:
if c == '\t':
@@ -70,24 +77,16 @@ def check_indent(cdline):
indent += 1
if c != ' ' and c != '\t':
break
-
return indent
-#srccom: raw comments in the source,including ''' and original indent
-
-
-def sampcd_extract_and_run(srccom,
- name,
- logf,
- htype="def",
- hname="",
- show_details=False):
- '''
+# srccom: raw comments in the source,including ''' and original indent
+def sampcd_extract_and_run(srccom, name, htype="def", hname=""):
+ """
Extract and run sample codes from source comment and
the result will be returned.
- As an ultimate result, this function returns a list of
+ As an ultimate result, this function returns a list of
status codes for each sample code (in top-down order)
found in srccom.
@@ -97,12 +96,12 @@ def sampcd_extract_and_run(srccom,
2:have sample code but format is wrong
1:no sample code
0:successful
- -1:no comments found
+ -1:no comments found
-2:in white list
-
+
there may be several examples in a source comment
so status deserves a list to contain the states.
- For instance, some API has three example codes,
+ For instance, some API has three example codes,
code 1 is successful, code 2 is error, code 3 is successful
so the list to return is [0,3,0]
@@ -110,111 +109,79 @@ def sampcd_extract_and_run(srccom,
srccom(str): the source comment of some API whose
example codes will be extracted and run.
name(str): the name of the API.
- logf(file): for logging the output in case they are
- flushed.
htype(str): the type of hint banners, def/class/method.
hname(str): the name of the hint banners , e.t. def hname.
- show_details(bool): Set it to False to print wrong sample
- codes only.
-
+
Returns:
list: the status code of all the sample codes found in srccom.
-
-
- '''
- def sampcd_header_print(name, sampcd, htype, hname, logf):
- '''
+
+ """
+
+ def sampcd_header_print(name, sampcd, htype, hname):
+ """
print hint banner headers.
-
+
Args:
name(str): the name of the API.
sampcd(str): sample code string
htype(str): the type of hint banners, def/class/method.
hname(str): the name of the hint banners , e.t. def hname.
- logf(file): for logging the output in case they are
flushed.
- '''
- print_header(logf, htype, hname)
-
- print "Sample code " + str(y) + " extracted for " + name + " :"
- print "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"
+ """
+ print_header(htype, hname)
+ print("Sample code ", str(y), " extracted for ", name, " :")
print(sampcd)
- print "----example code check----\n"
- print "executing sample code ....."
- print "execution result:"
- logf.write("\nSample code extracted for " + name + " :\n")
- logf.write("\n" + sampcd + "\n")
- logf.write("\n----example code check----\n")
- logf.write("\nexecuting sample code .....\n")
- logf.write("\nexecution result:\n")
+ print("----example code check----\n")
+ print("executing sample code .....")
+ print("execution result:")
sampcd_begins = find_all(srccom, " code-block:: python")
-
status = []
-
- if (len(sampcd_begins) == 0):
-
- print_header(logf, htype, hname)
+ if len(sampcd_begins) == 0:
+ print_header(htype, hname)
'''
detect sample codes using >>> to format
and consider this situation as wrong
'''
- if (srccom.find("Examples:") != -1):
- print "----example code check----\n"
- logf.write("\n----example code check----\n")
-
- if (srccom.find(">>>") != -1):
- logf.write(
- "Deprecated sample code style:\n\n Examples:\n\n >>>codeline\n >>>codeline\n\n\n "
- + "Please use '.. code-block:: python' to " +
- "format sample code.\n")
+ if srccom.find("Examples:") != -1:
+ print("----example code check----\n")
+ if srccom.find(">>>") != -1:
print(
- "Deprecated sample code style:\n\n Examples:\n\n >>>codeline\n >>>codeline\n\n\n "
- + "Please use '.. code-block:: python' to " +
+ "Deprecated sample code style:\n\n Examples:\n\n >>>codeline\n >>>codeline\n\n\n ",
+ "Please use '.. code-block:: python' to ",
"format sample code.\n")
status.append(2)
- print "status code for all sample codes in " + name + " : " + str(
- status)
-
+ print("status code for all sample codes in ", name, " : ",
+ str(status))
else:
- print "No sample code!\n"
- logf.write("\nNo sample code!\n")
+ print("No sample code!\n")
status.append(1)
- print "status code for all sample codes in " + name + " : " + str(
- status)
+ print("status code for all sample codes in ", name, " : ",
+ str(status))
for y in range(1, len(sampcd_begins) + 1):
-
sampcd_begin = sampcd_begins[y - 1]
sampcd = srccom[sampcd_begin + len(" code-block:: python") + 1:]
-
sampcd = sampcd.split("\n")
-
- #remove starting empty lines
+ # remove starting empty lines
while sampcd[0].replace(' ', '').replace('\t', '') == '':
sampcd.pop(0)
- #the mininmum indent, which is the indent of the first
- #non-empty line
+ # the minimum indent, which is the indent of the first
+ # non-empty line
min_indent = check_indent(sampcd[0])
-
sampcd_to_write = []
-
for i in range(0, len(sampcd)):
-
cdline = sampcd[i]
-
- #handle empty lines or those only with spaces/tabs
+ # handle empty lines or those only with spaces/tabs
if cdline.strip() == '':
continue
-
this_indent = check_indent(cdline)
- if (this_indent < min_indent):
+ if this_indent < min_indent:
break
-
else:
cdline = cdline.replace('\t', ' ')
sampcd_to_write.append(cdline[min_indent:])
@@ -224,94 +191,75 @@ def sampcd_extract_and_run(srccom,
sampcd = '\nimport os\n' + 'os.environ["CUDA_VISIBLE_DEVICES"] = ""\n' + sampcd
if sys.argv[1] == "gpu":
sampcd = '\nimport os\n' + 'os.environ["CUDA_VISIBLE_DEVICES"] = "0"\n' + sampcd
- sampcd += '\nprint ' + '\"' + name + ' sample code is executed successfully!\"\n'
+ sampcd += '\nprint(' + '\"' + name + ' sample code is executed successfully!\")'
- if (len(sampcd_begins) > 1):
+ if len(sampcd_begins) > 1:
tfname = name + "_example_" + str(y) + ".py"
else:
tfname = name + "_example" + ".py"
-
tempf = open("samplecode_temp/" + tfname, 'w')
tempf.write(sampcd)
tempf.close()
-
- cmd = ["python", "samplecode_temp/" + tfname]
+ if platform.python_version()[0] == "2":
+ cmd = ["python", "samplecode_temp/" + tfname]
+ elif platform.python_version()[0] == "3":
+ cmd = ["python3", "samplecode_temp/" + tfname]
+ else:
+ print("fail to parse python version!")
+ exit(1)
subprc = subprocess.Popen(
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output, error = subprc.communicate()
-
- msg = "".join(output)
- err = "".join(error)
-
- if (subprc.returncode != 0):
-
- print("\nSample code error found in " + name + ":\n")
- sampcd_header_print(name, sampcd, htype, hname, logf)
- print "subprocess return code: " + str(subprc.returncode)
- print("Error Raised from Sample Code " + name + " :\n")
- print err
- print msg
- logf.write("\nError Raised from Sample Code " + name + " :\n")
- logf.write("\n" + msg + "\n")
-
+ msg = "".join(output.decode(encoding='utf-8'))
+ err = "".join(error.decode(encoding='utf-8'))
+
+ if subprc.returncode != 0:
+ print("\nSample code error found in ", name, ":\n")
+ sampcd_header_print(name, sampcd, htype, hname)
+ print("subprocess return code: ", str(subprc.returncode))
+ print("Error Raised from Sample Code ", name, " :\n")
+ print(err)
+ print(msg)
status.append(3)
- print "status code for all sample codes in " + name + str(status)
- #It works!
+ print("status code for all sample codes in ", name, str(status))
+ # It works!
else:
status.append(0)
- if show_details:
- sampcd_header_print(name, sampcd, htype, hname, logf)
- print "subprocess return code: " + str(subprc.returncode)
- print msg
- logf.write("\n" + msg + "\n")
- print "status code for all sample codes in " + name + " : " + str(
- status)
-
- #msg is the returned code execution report
-
+ # msg is the returned code execution report
os.remove("samplecode_temp/" + tfname)
-
return status
def single_defcom_extract(start_from, srcls, is_class_begin=False):
- '''
+ """
to extract a def function/class/method comments body
- Args:
+ Args:
start_from(int): the line num of "def" header
srcls(list): the source file in lines
is_class_begin(bool): whether the start_from is a beginning a class. \
For a sole class body itself may end up with its method if it has no
docstring. But the body of \
a common def function can only be ended up by a none-indented def/class
-
+
Returns:
string : the extracted comment body, inclusive of its quote marks.
- '''
+ """
i = start_from
-
- fcombody = "" #def comment body
-
- comstart = -1 # the starting line index of comment mark "'''" or """"""
- #if it is not -1, it indicates the loop is in the comment body
+ fcombody = "" # def comment body
+ comstart = -1 # the starting line index of comment mark "'''" or """"""
+ # if it is not -1, it indicates the loop is in the comment body
comstyle = 0 # comment mark style ,comments quoted with ''' is coded as 1
# comments quoted with """ is coded as 2
-
for x in range(i + 1, len(srcls)):
-
if is_class_begin:
-
- if (srcls[x].replace('\t', ' ').startswith(' def ')):
+ if srcls[x].replace('\t', ' ').startswith(' def '):
break
-
- if ((srcls[x].startswith('def ') or srcls[x].startswith('class '))):
+ if srcls[x].startswith('def ') or srcls[x].startswith('class '):
break
-
else:
-
if (comstart == -1 and srcls[x].replace(" ", '').replace(
"\t", '').replace("\n", '').startswith("\"\"\"")):
comstart = x
@@ -321,7 +269,6 @@ def single_defcom_extract(start_from, srcls, is_class_begin=False):
srcls[x].replace(" ", '').replace("\t", '').replace(
"\n", '').startswith("\"\"\"")):
break
-
if (comstart == -1 and srcls[x].replace(" ", '').replace(
"\t", '').replace("\n", '').startswith("\'\'\'")):
comstart = x
@@ -332,99 +279,61 @@ def single_defcom_extract(start_from, srcls, is_class_begin=False):
"\n", '').startswith("\'\'\'")):
break
if (comstart !=
- -1): #when the comments start, begin to add line to fcombody
+ -1): # when the comments start, begin to add line to fcombody
fcombody += srcls[x]
-
return fcombody
-def print_header(logf, htype, name):
-
- print htype + " name:" + name
- print "-----------------------"
-
- logf.write("\n\n" + htype + " name:" + name + "\n")
- logf.write("-----------------------\n")
-
-
-def srcf_print(srcfile):
+def print_header(htype, name):
+ print(htype, " name:", name)
+ print("-----------------------")
- print "source file name:" + srcfile.name
- print "---------------------------------------------------"
- logf.write("source file name:" + srcfile.name + "\n")
- logf.write("---------------------------------------------------\n\n")
-
-
-def show_alllist(alllist):
-
- print "__all__:" + str(alllist) + "\n"
- logf.write("__all__:" + str(alllist) + "\n\n")
-
-
-def srccoms_extract(srcfile, logf, status_all, wlist, show_details):
- '''
+def srccoms_extract(srcfile, status_all, wlist):
+ """
Given a source file ``srcfile``, this function will
extract its API(doc comments) and run sample codes in the
API.
Args:
srcfile(file): the source file
- logf(file): log recording file
status_all(dict): record all the sample code execution states.
wlist(list): white list
- show_details(bool): if show_details is True, the whole process will be printed for you
- to debug it locally
Returns:
string: the length of __all__ list in srcfile versus the exact number of
analysed API to make sure no API is missed in this srcfile and it
is useful for statistic practices.
- '''
+ """
srcc = srcfile.read()
- #2. get defs and classes header line number
- #set file pointer to its beginning
+ # 2. get defs and classes header line number
+ # set file pointer to its beginning
srcfile.seek(0, 0)
- srcls = srcfile.readlines() #source lines
-
- if show_details:
- srcf_print(srcfile)
+ srcls = srcfile.readlines() # source lines
- #1. fetch__all__ list
+ # 1. fetch__all__ list
allidx = srcc.find("__all__")
- if (allidx != -1):
-
+ if allidx != -1:
alllist = []
-
- #get all list for layers/ops.py
- if (srcfile.name.find("ops.py") != -1):
-
+ # get all list for layers/ops.py
+ if srcfile.name.find("ops.py") != -1:
for ai in range(0, len(srcls)):
-
- if (srcls[ai].startswith("__all__")):
-
+ if srcls[ai].startswith("__all__"):
lb = srcls[ai].find('[')
rb = srcls[ai].find(']')
- if (lb == -1):
+ if lb == -1:
continue
allele = srcls[ai][lb + 1:rb].replace("'", '').replace(
" ", '').replace("\"", '')
-
alllist.append(allele)
-
if '' in alllist:
alllist.remove('')
-
- if show_details:
- show_alllist(alllist)
-
else:
alllist_b = allidx + len("__all__")
-
allstr = srcc[alllist_b + srcc[alllist_b:].find("[") + 1:alllist_b +
srcc[alllist_b:].find("]")]
allstr = allstr.replace("\n", '').replace(" ", '').replace(
@@ -432,295 +341,184 @@ def srccoms_extract(srcfile, logf, status_all, wlist, show_details):
alllist = allstr.split(',')
if '' in alllist:
alllist.remove('')
-
- if show_details:
- show_alllist(alllist)
-
api_alllist_count = len(alllist)
api_count = 0
-
handled = []
- #get src contents in layers/ops.py
- if (srcfile.name.find("ops.py") != -1):
-
+ # get src contents in layers/ops.py
+ if srcfile.name.find("ops.py") != -1:
for i in range(0, len(srcls)):
-
if srcls[i].find("__doc__") != -1:
-
opname = srcls[i][:srcls[i].find("__doc__") - 1]
-
if opname in wlist:
-
status_all[srcfile.name + '/' + opname] = [-2]
-
- if show_details:
- print_header(logf, "def", opname)
- print opname + " is in white list, thus skipped"
- logf.write("\n" + opname +
- " is in white list, thus skipped\n")
- print status_all[srcfile.name + '/' + opname]
- logf.write("\n" + "execution status" + str(
- status_all[srcfile.name + '/' + opname]) + "\n")
-
continue
-
comstart = i
for j in range(i, len(srcls)):
- if (srcls[j].find("\"\"\"") != -1):
+ if srcls[j].find("\"\"\"") != -1:
comstart = i
-
opcom = ""
for j in range(comstart + 1, len(srcls)):
opcom += srcls[j]
- if (srcls[j].find("\"\"\"") != -1):
+ if srcls[j].find("\"\"\"") != -1:
break
-
- status = sampcd_extract_and_run(opcom, opname, logf, "def",
- opname, show_details)
+ status = sampcd_extract_and_run(opcom, opname, "def",
+ opname)
api_count += 1
status_all[srcfile.name + '/' + opname] = status
-
handled.append(
- opname) #ops.py also has normal formatted functions
- #use list 'handled' to mark the functions have been handled here
- #which will be ignored in the following step
+ opname) # ops.py also has normal formatted functions
+ # use list 'handled' to mark the functions have been handled here
+ # which will be ignored in the following step
for i in range(0, len(srcls)):
-
if srcls[i].startswith(
- 'def '): #a function header is detected in line i
-
+ 'def '): # a function header is detected in line i
f_header = srcls[i].replace(" ", '')
- fn = f_header[len('def'):f_header.find('(')] #function name
-
+ fn = f_header[len('def'):f_header.find('(')] # function name
if fn in handled:
continue
-
if fn in alllist:
-
api_count += 1
-
if fn in wlist or fn + "@" + srcfile.name in wlist:
-
status_all[srcfile.name + '/' + fn] = [-2]
-
- if show_details:
- print_header(logf, "def", fn)
- print fn + " is in white list, thus skipped"
- logf.write("\n" + fn +
- " is in white list, thus skipped\n")
- print status_all[srcfile.name + '/' + fn]
- logf.write("\n" + "execution status" + str(
- status_all[srcfile.name + '/' + fn]) + "\n")
-
continue
-
fcombody = single_defcom_extract(i, srcls)
- if (fcombody == ""): #if no comment
- print_header(logf, "def", fn)
- print "WARNING: no comments in function " + fn + ", but it deserves."
- logf.write("no comments in function " + fn + "\n\n")
+ if fcombody == "": # if no comment
+ print_header("def", fn)
+ print("WARNING: no comments in function ", fn,
+ ", but it deserves.")
status_all[srcfile.name + '/' + fn] = [-1]
- print status_all[srcfile.name + '/' + fn]
- logf.write("\n" + "execution status" + str(status_all[
- srcfile.name + '/' + fn]) + "\n")
-
+ print(status_all[srcfile.name + '/' + fn])
continue
else:
- status = sampcd_extract_and_run(fcombody, fn, logf,
- "def", fn, show_details)
+ status = sampcd_extract_and_run(fcombody, fn, "def", fn)
status_all[srcfile.name + '/' + fn] = status
-
- else:
- if show_details:
- print_header(logf, "def", fn)
- print fn + " not in __all__ list"
- logf.write(fn + " not in __all__ list\n\n")
-
if srcls[i].startswith('class '):
-
c_header = srcls[i].replace(" ", '')
- cn = c_header[len('class'):c_header.find('(')] #class name
-
+ cn = c_header[len('class'):c_header.find('(')] # class name
if cn in handled:
continue
-
if cn in alllist:
-
api_count += 1
-
if cn in wlist or cn + "@" + srcfile.name in wlist:
-
status_all[srcfile.name + '/' + cn] = [-2]
-
- if show_details:
-
- print cn + " is in white list, thus skipped"
- logf.write("\n" + cn +
- " is in white list, thus skipped\n")
-
- print status_all[srcfile.name + '/' + cn]
- logf.write("\n" + "execution status" + str(
- status_all[srcfile.name + '/' + cn]) + "\n")
-
continue
-
- #class comment
+ # class comment
classcom = single_defcom_extract(i, srcls, True)
-
- if (classcom != ""):
-
- status = sampcd_extract_and_run(
- classcom, cn, logf, "class", cn, show_details)
+ if classcom != "":
+ status = sampcd_extract_and_run(classcom, cn, "class",
+ cn)
status_all[srcfile.name + '/' + cn] = status
-
else:
- print "WARNING: no comments in class itself " + cn + ", but it deserves.\n"
- logf.write("no comments in class itself " + cn +
- "\n\n\n")
+ print("WARNING: no comments in class itself ", cn,
+ ", but it deserves.\n")
status_all[srcfile.name + '/' + cn] = [-1]
- print status_all[srcfile.name + '/' + cn]
- logf.write("\n" + "execution status" + str(status_all[
- srcfile.name + '/' + cn]) + "\n")
-
- #handling methods in class bodies
+ print(status_all[srcfile.name + '/' + cn])
+ # handling methods in class bodies
for x in range(
i + 1,
- len(srcls)): #from the next line of class header
-
+ len(srcls)): # from the next line of class header
if (srcls[x].startswith('def ') or
srcls[x].startswith('class ')):
break
else:
- #member method def header
+ # member method def header
srcls[x] = srcls[x].replace('\t', ' ')
if (srcls[x].startswith(
- ' def ')): #detect a mehtod header..
-
+ ' def ')): # detect a mehtod header..
thisl = srcls[x]
indent = len(thisl) - len(thisl.lstrip())
mn = thisl[indent + len('def '):thisl.find(
- '(')] #method name
-
- name = cn + "." + mn #full name
-
+ '(')] # method name
+ name = cn + "." + mn # full name
if mn.startswith('_'):
-
- if show_details:
-
- print mn + " is hidden, not visible to users\n"
- logf.write(
- "\n" + mn +
- " is hidden, not visible to users\n")
-
continue
-
if name in wlist or name + "@" + srcfile.name in wlist:
-
status_all[srcfile.name + '/' + name] = [-2]
-
- if show_details:
-
- print name + " is in white list, thus skipped"
- logf.write(
- "\n" + name +
- " is in white list, thus skipped\n")
- print status_all[srcfile.name + '/' +
- name]
- logf.write(
- "\n" + "execution status" + str(
- status_all[srcfile.name + '/' +
- name]) + "\n")
-
continue
-
- thismethod = [] #method body lines
- thismethod.append(thisl[indent:])
-
- #get all the lines of a single method body
- #into thismethod(list)
- #and send it to single_defcom_extract
+ thismethod = [thisl[indent:]
+ ] # method body lines
+ # get all the lines of a single method body
+ # into thismethod(list)
+ # and send it to single_defcom_extract
for y in range(x + 1, len(srcls)):
srcls[y] = srcls[y].replace('\t', ' ')
if (srcls[y].startswith('def ') or
srcls[y].startswith('class ')):
- #end of method
+ # end of method
break
- elif (srcls[y].startswith(' def ')):
- #end of method
+ elif srcls[y].startswith(' def '):
+ # end of method
break
else:
thismethod.append(srcls[y][indent:])
-
thismtdcom = single_defcom_extract(0,
thismethod)
-
- if (thismtdcom != ""):
+ if thismtdcom != "":
status = sampcd_extract_and_run(
- thismtdcom, name, logf, "method", name,
- show_details)
+ thismtdcom, name, "method", name)
status_all[srcfile.name + '/' +
name] = status
-
- else:
-
- if show_details:
- print "no comments in method " + name + "\n"
- logf.write("no comments in method " +
- name + "\n\n\n")
- status_all[srcfile.name + '/' +
- name] = [-1]
- print status_all[srcfile.name + '/' +
- name]
- logf.write(
- "\n" + "execution status" + str(
- status_all[srcfile.name + '/' +
- name]) + "\n")
-
- else:
- if show_details:
- print cn + " is not in __all__ list"
- logf.write(cn + " is not in __all__ list\n\n")
-
return [
srcfile.name + " all list length: " + str(api_alllist_count),
"analysed api count: " + str(api_count)
]
+def test(file_list):
+ for file in file_list:
+ src = open(file, 'r')
+ counts = srccoms_extract(src, status_all, wlist)
+ src.close()
+
+
'''
Important constant lists:
filenames : the modules pending for check .
wlist : a list of API that should not trigger the example check .
It is composed of wlist_temp + wlist_inneed + wlist_ignore.
- show_details: a boolean value to indicate whether it should be run
- in debugging mode.
status_all: a status list containing all the execution status of all
APIs
srcfile: the source .py code file
'''
filenames = [
- "layers/control_flow.py", "layers/io.py", "layers/nn.py", "layers/ops.py",
- "layers/tensor.py", "layers/learning_rate_scheduler.py",
- "layers/detection.py", "layers/metric_op.py"
+ "../python/paddle/fluid/layers/control_flow.py",
+ "../python/paddle/fluid/layers/io.py",
+ "../python/paddle/fluid/layers/nn.py",
+ "../python/paddle/fluid/layers/ops.py",
+ "../python/paddle/fluid/layers/tensor.py",
+ "../python/paddle/fluid/layers/learning_rate_scheduler.py",
+ "../python/paddle/fluid/layers/detection.py",
+ "../python/paddle/fluid/layers/metric_op.py"
]
filenames += [
- "dygraph/layers.py", "dygraph/base.py", "dygraph/nn.py",
- "dygraph/tracer.py", "dygraph/profiler.py", "dygraph/parallel.py",
- "dygraph/checkpoint.py", "dygraph/learning_rate_scheduler.py",
- "dygraph/backward_strategy.py"
+ "../python/paddle/fluid/dygraph/layers.py",
+ "../python/paddle/fluid/dygraph/base.py",
+ "../python/paddle/fluid/dygraph/nn.py",
+ "../python/paddle/fluid/dygraph/tracer.py",
+ "../python/paddle/fluid/dygraph/profiler.py",
+ "../python/paddle/fluid/dygraph/parallel.py",
+ "../python/paddle/fluid/dygraph/checkpoint.py",
+ "../python/paddle/fluid/dygraph/learning_rate_scheduler.py",
+ "../python/paddle/fluid/dygraph/backward_strategy.py"
]
-
filenames += [
- "data_feeder.py", "dataset.py", "clip.py", "metrics.py", "executor.py",
- "initializer.py", "io.py", "nets.py", "optimizer.py", "profiler.py",
- "regularizer.py", "backward.py", "average.py", "unique_name.py",
- "framework.py", "evaluator.py", "param_attr.py"
+ "../python/paddle/fluid/data_feeder.py",
+ "../python/paddle/fluid/dataset.py", "../python/paddle/fluid/clip.py",
+ "../python/paddle/fluid/metrics.py", "../python/paddle/fluid/executor.py",
+ "../python/paddle/fluid/initializer.py", "../python/paddle/fluid/io.py",
+ "../python/paddle/fluid/nets.py", "../python/paddle/fluid/optimizer.py",
+ "../python/paddle/fluid/profiler.py",
+ "../python/paddle/fluid/regularizer.py",
+ "../python/paddle/fluid/backward.py", "../python/paddle/fluid/average.py",
+ "../python/paddle/fluid/unique_name.py",
+ "../python/paddle/fluid/framework.py",
+ "../python/paddle/fluid/evaluator.py",
+ "../python/paddle/fluid/param_attr.py"
]
-
wlist_inneed = [
"append_LARS", "BuildStrategy.debug_graphviz_path",
"BuildStrategy.enable_sequential_execution",
@@ -752,7 +550,6 @@ wlist_inneed = [
'StaticRNN.output', "cuda_places", "CUDAPinnedPlace", "CUDAPlace",
"Program.parse_from_string"
]
-
wlist_temp = [
'ChunkEvaluator',
'EditDistance',
@@ -851,7 +648,6 @@ wlist_ignore = [
'Precision.update', 'WeightedAverage.eval', 'Conv3D.forward',
'Embedding.forward', 'Recall.eval', 'FC.forward', 'While.block'
]
-
# only white on CPU
gpu_not_white = [
"deformable_conv", "cuda_places", "CUDAPinnedPlace", "CUDAPlace",
@@ -861,146 +657,65 @@ gpu_not_white = [
wlist = wlist_temp + wlist_inneed + wlist_ignore
if len(sys.argv) < 2:
- print "Error: inadequate number of arguments"
+ print("Error: inadequate number of arguments")
print('''If you are going to run it on
"CPU: >>> python sampcd_processor.py cpu
"GPU: >>> python sampcd_processor.py gpu
''')
sys.exit("lack arguments")
-
else:
-
- show_details = False
-
if sys.argv[1] == "gpu":
for _gnw in gpu_not_white:
wlist.remove(_gnw)
elif sys.argv[1] != "cpu":
- print("Unrecognized argument:'" + sys.argv[1] + "' , 'cpu' or 'gpu' is "
- + "desired\n")
+ print("Unrecognized argument:'", sys.argv[1], "' , 'cpu' or 'gpu' is ",
+ "desired\n")
sys.exit("Invalid arguments")
-
- if len(sys.argv) == 3:
- if sys.argv[2] == "sd":
- show_details = True
- else:
- print("Unrecognized argument:'" + sys.argv[2] + "' , 'sd' is " +
- "desired\n")
- sys.exit("Invalid arguments")
-
- print("* * * * * * * * * * * * * * * * * * * * * * * *\n" +
- "* *\n" +
- "* API check -- Example Code Cheker *\n" +
- "* *\n" +
- "* *\n" +
- "* This process is meant to check *\n" +
- "* all example codes per CI to ensure *\n" +
- "* the example codes can be run successfully *\n" +
- "* *\n" +
- "* *\n" +
- "* Refer to the comments for detailed *\n" +
- "* introduction *\n" +
- "* *\n" +
- "* *\n" +
- "* * * * * * * * * * * * * * * * * * * * * * * *\n")
-
+ print("API check -- Example Code")
+ print("sample_test running under python", platform.python_version())
status_all = {}
-
- #a file to record the terminal output
- logf = open("example-code-check-log.txt", 'w')
-
# a temp directory to store temporary sample code file
- # subprocess needs a single file to run the code
-
+ # subprocess needs a single file to run the code
if not os.path.isdir("./samplecode_temp"):
os.mkdir("./samplecode_temp")
- to_check = filenames
- for filename in to_check:
-
- srcfile = open(filename, 'r')
-
- counts = srccoms_extract(srcfile, logf, status_all, wlist, show_details)
-
- if show_details:
- logf.write("\n\n" + str(counts) + "\n\n")
-
- srcfile.close()
+ one_part_filenum = int(math.ceil(len(filenames) / 10))
+ divided_file_list = [
+ filenames[i:i + one_part_filenum]
+ for i in range(0, len(filenames), one_part_filenum)
+ ]
+ po = multiprocessing.Pool(10)
+ for file_list in divided_file_list:
+ po.apply_async(test, (file_list, ))
+ po.close()
+ po.join()
# clear temp files
for root, dirs, files in os.walk("./samplecode_temp"):
for fntemp in files:
os.remove("./samplecode_temp/" + fntemp)
-
os.rmdir("./samplecode_temp")
status_groups = {-2: [], -1: [], 0: [], 1: [], 2: [], 3: []}
ci_pass = True
-
for key in status_all:
statusl = status_all[key]
for ele in statusl:
- if (ele != 0 and ele != -2 and ele != -1):
+ if ele != 0 and ele != -2 and ele != -1:
ci_pass = False
break
-
if len(statusl) == 1:
status_groups[statusl[0]].append(key)
else:
for u in range(0, len(statusl)):
status_groups[statusl[u]].append(key + '_' + str(u + 1))
-
- logf.close()
-
- print(
- "\n\n------------------End of the Check-------------------------------------------\n\n"
- )
-
+ print("----------------End of the Check--------------------")
errorapisl = status_groups[1] + status_groups[2] + status_groups[3]
if len(errorapisl) > 0:
- print "Error raised from: " + str(errorapisl)
-
+ print("Error raised from: ", str(errorapisl))
if not ci_pass:
-
- print(
- "\nOh no.. Mistakes found in sample codes, refer to the log for details\n\n"
- )
- print('''
-- How to run it locally?
-
- Simply put this script under directory:
-
- Paddle/python/paddle/fluid/
-
- and run in python 2.7 (as some interfaces of subprocess may
- not work in python 3)
-
- You must specify the device type to run the sample code on:
-
- CPU: >>> python sampcd_processor.py cpu
- GPU: >>> python sampcd_processor.py gpu
-
-- How to debug?
-
- This script has an option for showing the details of
- the execution status:
-
- >>> python sampcd_processor.py cpu sd
-
-- NOTE:
-
- Please ensure your are using
-
- .. code-block:: python
-
- [sample code starts here]
-
- ONLY 1 BLANKSPACE between '::' and 'python'
-
- ''')
-
+ print("Mistakes found in sample codes")
exit(1)
else:
-
- print "Sample code check is successful!"
+ print("Sample code check is successful!")