Skip to content

P
Paddle

PaddlePaddle / Paddle
大约 1 年 前同步成功

通知 2298
Star 20931
Fork 5422
P
Paddle
未验证 提交 ef8671e5 编写于 作者: R Ren Wei (任卫) 提交者： GitHub
浏览文件
操作

print the real name for Functions instead of the ArgSpec (#32379) 

* print the real name for Functions instead of the ArgSpec

class function method

* some API's name is not __module__ + __name__

so, we disard them temporarily.

* update the logging format for console

* ommit the top level of paddle package.

* these APIs have been removed.

test=document_fix

* Another Error occerd

* print_signatures.py 's stdout is redirect to spec file, so should not print any other info.

so sad.

* print the error msg to stderr

* disable the __init__ magic method

* update unittest for sampcd_processor.py

update unittest for sampcd_processor.py

* PR-CI-APPROVAL 's python interpreter name is not 'python3'.

it's a python3.9;
it does not have paddle installed yet.

此句在CI流水线竟然不可以用。报python3找不到
此句在CI流水线竟然不可以用。因为环境没有安装paddle

* testing only extract api from __all__

paddle module(the top module) does not have __add__
test=document_fix

* should import paddle here

* update the mechanism of extractiong and executing for the sample-codes test.

更新抽取代码和执行代码的逻辑
优化输出打印

* good code style
上级 f8caa587
隐藏空白更改
内联 并排
Showing with 156 addition and 157 deletion
tools/print_signatures.py
浏览文件 @ ef8671e5
......@@ -27,19 +27,12 @@ import pydoc
import hashlib
import six
import functools
import logging
import paddle
member_dict = collections.OrderedDict()
visited_modules = set()
# APIs that should not be printed into API.spec
omitted_list = [
"paddle.fluid.LoDTensor.set", # Do not know why it should be omitted
"paddle.fluid.io.ComposeNotAligned",
"paddle.fluid.io.ComposeNotAligned.__init__",
]
def md5(doc):
hash = hashlib.md5()
......@@ -74,13 +67,28 @@ def format_spec(spec):
def queue_dict(member, cur_name):
if cur_name in omitted_list:
return
doc_md5 = md5(member.__doc__)
if inspect.isclass(member):
if cur_name != 'paddle':
try:
eval(cur_name)
except (AttributeError, NameError, SyntaxError) as e:
print(
"Error({}) occurred when `eval({})`, discard it.".format(
str(e), cur_name),
file=sys.stderr)
return
if (inspect.isclass(member) or inspect.isfunction(member) or
inspect.ismethod(member)) and hasattr(
member, '__module__') and hasattr(member, '__name__'):
args = member.__module__ + "." + member.__name__
try:
eval(args)
except (AttributeError, NameError, SyntaxError) as e:
print(
"Error({}) occurred when `eval({})`, discard it for {}.".format(
str(e), args, cur_name),
file=sys.stderr)
return
else:
try:
args = inspect.getargspec(member)
......@@ -95,6 +103,7 @@ def queue_dict(member, cur_name):
if not has_type_error:
args = format_spec(args)
doc_md5 = md5(member.__doc__)
member_dict[cur_name] = "({}, ('document', '{}'))".format(args, doc_md5)
......@@ -106,8 +115,7 @@ def visit_member(parent_name, member, member_name=None):
if inspect.isclass(member):
queue_dict(member, cur_name)
for name, value in inspect.getmembers(member):
if hasattr(value, '__name__') and (not name.startswith("_") or
name == "__init__"):
if hasattr(value, '__name__') and not name.startswith("_"):
visit_member(cur_name, value)
elif inspect.ismethoddescriptor(member):
return
......@@ -149,11 +157,14 @@ def visit_all_module(mod):
return
visited_modules.add(mod)
for member_name in (
name
for name in (mod.__all__ if hasattr(mod, "__all__") else dir(mod))
if not name.startswith("_")):
if hasattr(mod, "__all__"):
member_names = (name for name in mod.__all__
if not name.startswith("_"))
elif mod_name == 'paddle':
member_names = dir(mod)
else:
return
for member_name in member_names:
instance = getattr(mod, member_name, None)
if instance is None:
continue
......@@ -168,17 +179,19 @@ def visit_all_module(mod):
visit_all_module(instance)
else:
if member_name != instance.__name__:
logging.warn(
print(
"Found alias API, alias name is: {}, original name is: {}".
format(member_name, instance.__name__))
format(member_name, instance.__name__),
file=sys.stderr)
visit_member(mod.__name__, instance, member_name)
else:
visit_member(mod.__name__, instance)
modules = sys.argv[1].split(",")
for m in modules:
visit_all_module(importlib.import_module(m))
if __name__ == '__main__':
modules = sys.argv[1].split(",")
for m in modules:
visit_all_module(importlib.import_module(m))
for name in member_dict:
print(name, member_dict[name])
for name in member_dict:
print(name, member_dict[name])
tools/sampcd_processor.py
浏览文件 @ ef8671e5
......@@ -19,8 +19,8 @@ import multiprocessing
import math
import platform
import inspect
import paddle
import paddle.fluid
#import paddle
#import paddle.fluid
import json
import argparse
import shutil
......@@ -44,9 +44,7 @@ if logger.handlers:
else:
console = logging.StreamHandler()
logger.addHandler(console)
console.setFormatter(
logging.Formatter(
"%(asctime)s - %(funcName)s:%(lineno)d - %(levelname)s - %(message)s"))
console.setFormatter(logging.Formatter("%(message)s"))
RUN_ON_DEVICE = 'cpu'
GPU_ID = 0
......@@ -125,29 +123,41 @@ def sampcd_extract_and_run(srccom, name, htype="def", hname=""):
name(str): the name of the API.
msg(str): messages
"""
global GPU_ID, RUN_ON_DEVICE, SAMPLECODE_TEMPDIR
sample_code_filenames = sampcd_extract_to_file(srccom, name, htype, hname)
if not sample_code_filenames:
return False, name, 'No sample code!'
results = []
msgs = []
for tfname in sample_code_filenames:
result, msg = execute_samplecode_test(tfname)
results.append(result)
msgs.append(msg)
if not all(results):
failed_fn = []
for i, result in enumerate(results):
if not result:
failed_fn.append(sample_code_filenames[i])
return False, name, 'failed sample codes: ' + ','.join(failed_fn)
return True, name, 'success!'
def sampcd_extract_to_file(srccom, name, htype="def", hname=""):
"""
Extract sample codes from __doc__, and write them to files.
result = True
msg = None
Args:
srccom(str): the source comment of some API whose
example codes will be extracted and run.
name(str): the name of the API.
htype(str): the type of hint banners, def/class/method.
hname(str): the name of the hint banners , e.t. def hname.
def sampcd_header_print(name, sampcd, htype, hname):
"""
print hint banner headers.
Args:
name(str): the name of the API.
sampcd(str): sample code string
htype(str): the type of hint banners, def/class/method.
hname(str): the name of the hint banners , e.t. def hname.
flushed.
"""
print(htype, " name:", hname)
print("-----------------------")
print("Sample code ", str(y), " extracted for ", name, " :")
print(sampcd)
print("----example code check----\n")
print("executing sample code .....")
print("execution result:")
Returns:
sample_code_filenames(list of str)
"""
global GPU_ID, RUN_ON_DEVICE, SAMPLECODE_TEMPDIR
sampcd_begins = find_all(srccom, " code-block:: python")
if len(sampcd_begins) == 0:
......@@ -161,11 +171,11 @@ def sampcd_extract_and_run(srccom, name, htype="def", hname=""):
"Deprecated sample code style:\n\n Examples:\n\n >>>codeline\n >>>codeline\n\n\n ",
"Please use '.. code-block:: python' to ",
"format sample code.\n")
result = False
return []
else:
print("Error: No sample code!\n")
result = False
return []
sample_code_filenames = []
for y in range(1, len(sampcd_begins) + 1):
sampcd_begin = sampcd_begins[y - 1]
sampcd = srccom[sampcd_begin + len(" code-block:: python") + 1:]
......@@ -200,37 +210,50 @@ def sampcd_extract_and_run(srccom, name, htype="def", hname=""):
tfname = os.path.join(SAMPLECODE_TEMPDIR, '{}_example{}'.format(
name, '.py' if len(sampcd_begins) == 1 else '_{}.py'.format(y)))
logging.info('running %s', tfname)
with open(tfname, 'w') as tempf:
tempf.write(sampcd)
if platform.python_version()[0] == "2":
cmd = ["python", tfname]
elif platform.python_version()[0] == "3":
cmd = ["python3", tfname]
else:
print("Error: fail to parse python version!")
result = False
exit(1)
sample_code_filenames.append(tfname)
return sample_code_filenames
subprc = subprocess.Popen(
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output, error = subprc.communicate()
msg = "".join(output.decode(encoding='utf-8'))
err = "".join(error.decode(encoding='utf-8'))
if subprc.returncode != 0:
print("\nSample code error found in ", name, ":\n")
sampcd_header_print(name, sampcd, htype, hname)
print("subprocess return code: ", str(subprc.returncode))
print("Error Raised from Sample Code ", name, " :\n")
print(err)
print(msg)
logging.warning('%s error: %s', tfname, err)
logging.warning('%s msg: %s', tfname, msg)
result = False
# msg is the returned code execution report
def execute_samplecode_test(tfname):
result = True
msg = None
if platform.python_version()[0] in ["2", "3"]:
cmd = [sys.executable, tfname]
else:
print("Error: fail to parse python version!")
result = False
exit(1)
logging.info('running %s', tfname)
print("\n----example code check----")
print("executing sample code .....", tfname)
subprc = subprocess.Popen(
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output, error = subprc.communicate()
msg = "".join(output.decode(encoding='utf-8'))
err = "".join(error.decode(encoding='utf-8'))
if subprc.returncode != 0:
print("Sample code error found in ", tfname, ":")
print("-----------------------")
print(open(tfname).read())
print("-----------------------")
print("subprocess return code: ", str(subprc.returncode))
print("Error Raised from Sample Code ", tfname, " :")
print(err)
print(msg)
print("----example code check failed----\n")
logging.warning('%s error: %s', tfname, err)
logging.warning('%s msg: %s', tfname, msg)
result = False
else:
print("----example code check success----\n")
return result, name, msg
# msg is the returned code execution report
return result, tfname, msg
def single_defcom_extract(start_from, srcls, is_class_begin=False):
......@@ -326,7 +349,7 @@ def srccoms_extract(srcfile, wlist, methods):
if allidx != -1:
alllist = []
# get all list for layers/ops.py
if srcfile.name.find("ops.py") != -1:
if srcfile.name.find("fluid/layers/ops.py") != -1:
for ai in range(0, len(srcls)):
if srcls[ai].startswith("__all__"):
lb = srcls[ai].find('[')
......@@ -352,7 +375,7 @@ def srccoms_extract(srcfile, wlist, methods):
api_count = 0
handled = []
# get src contents in layers/ops.py
if srcfile.name.find("ops.py") != -1:
if srcfile.name.find("fluid/layers/ops.py") != -1:
for i in range(0, len(srcls)):
opname = None
opres = re.match(r"^(\w+)\.__doc__", srcls[i])
......@@ -536,25 +559,24 @@ def run_a_test(tc_filename):
def get_filenames():
'''
this function will get the modules that pending for check.
this function will get the sample code files that pending for check.
Returns:
list: the modules pending for check .
dict: the sample code files pending for check .
'''
filenames = []
global methods # write
global whl_error
methods = []
import paddle
whl_error = []
get_incrementapi()
API_spec = API_DIFF_SPEC_FN
with open(API_spec) as f:
all_sample_code_filenames = {}
with open(API_DIFF_SPEC_FN) as f:
for line in f.readlines():
api = line.replace('\n', '')
try:
module = eval(api).__module__
api_obj = eval(api)
except AttributeError:
whl_error.append(api)
continue
......@@ -562,47 +584,12 @@ def get_filenames():
logger.warning('line:%s, api:%s', line, api)
# paddle.Tensor.<lambda>
continue
if len(module.split('.')) > 1:
filename = '../python/'
# work for .so?
module_py = '%s.py' % module.split('.')[-1]
for i in range(0, len(module.split('.')) - 1):
filename = filename + '%s/' % module.split('.')[i]
filename = filename + module_py
else:
filename = ''
logger.warning("WARNING: Exception in getting api:%s module:%s",
api, module)
if filename in filenames:
continue
elif not filename:
logger.warning('filename invalid: %s', line)
continue
elif not os.path.exists(filename):
logger.warning('file not exists: %s', filename)
continue
else:
filenames.append(filename)
# get all methods
method = ''
if inspect.isclass(eval(api)):
name = api.split('.')[-1]
elif inspect.isfunction(eval(api)):
name = api.split('.')[-1]
elif inspect.ismethod(eval(api)):
name = '%s.%s' % (api.split('.')[-2], api.split('.')[-1])
else:
name = ''
logger.warning(
"WARNING: Exception when getting api:%s, line:%s", api,
line)
for j in range(2, len(module.split('.'))):
method = method + '%s.' % module.split('.')[j]
method = method + name
if method not in methods:
methods.append(method)
os.remove(API_spec)
return filenames
if hasattr(api_obj, '__doc__') and api_obj.__doc__:
sample_code_filenames = sampcd_extract_to_file(api_obj.__doc__,
api)
for tfname in sample_code_filenames:
all_sample_code_filenames[tfname] = api
return all_sample_code_filenames
def get_api_md5(path):
......@@ -744,14 +731,6 @@ if __name__ == '__main__':
if len(filenames) == 0 and len(whl_error) == 0:
logger.info("-----API_PR.spec is the same as API_DEV.spec-----")
exit(0)
rm_file = []
for f in filenames:
for w_file in wlist_file:
if f.startswith(w_file):
rm_file.append(f)
filenames.remove(f)
if len(rm_file) != 0:
logger.info("REMOVE white files: %s", rm_file)
logger.info("API_PR is diff from API_DEV: %s", filenames)
threads = multiprocessing.cpu_count()
......@@ -759,7 +738,7 @@ if __name__ == '__main__':
threads = args.threads
po = multiprocessing.Pool(threads)
# results = po.map_async(test, divided_file_list)
results = po.map_async(run_a_test, filenames)
results = po.map_async(execute_samplecode_test, filenames.keys())
po.close()
po.join()
......
tools/test_sampcd_processor.py
浏览文件 @ ef8671e5
......@@ -271,6 +271,10 @@ class Test_get_wlist(unittest.TestCase):
class Test_srccoms_extract(unittest.TestCase):
def setUp(self):
self.tmpDir = tempfile.mkdtemp()
print('tmpDir=', self.tmpDir)
self.opsDir = os.path.join(self.tmpDir, 'fluid/layers')
os.makedirs(self.opsDir)
sys.path.append(self.opsDir)
sys.path.append(self.tmpDir)
self.api_pr_spec_filename = os.path.abspath(
os.path.join(os.getcwd(), "..", 'paddle/fluid/API_PR.spec'))
......@@ -283,7 +287,7 @@ class Test_srccoms_extract(unittest.TestCase):
]))
def tearDown(self):
sys.path.remove(self.tmpDir)
#sys.path.remove(self.tmpDir)
shutil.rmtree(self.tmpDir)
os.remove(self.api_pr_spec_filename)
......@@ -305,9 +309,11 @@ def exp():
add_sample_code(globals()["exp"], r"""
Examples:
.. code-block:: python
import paddle
x = paddle.to_tensor([-0.4, -0.2, 0.1, 0.3])
out = paddle.exp(x)
# import paddle
# x = paddle.to_tensor([-0.4, -0.2, 0.1, 0.3])
# out = paddle.exp(x)
out = [0.67032005, 0.81873075, 1.10517092, 1.34985881]
print(out)
# [0.67032005 0.81873075 1.10517092 1.34985881]
""")
......@@ -332,7 +338,7 @@ add_sample_code(globals()["two_plus_two"], """
print(2+2)
""")
'''
pyfilename = os.path.join(self.tmpDir, 'ops.py')
pyfilename = os.path.join(self.opsDir, 'ops.py')
with open(pyfilename, 'w') as pyfile:
pyfile.write(filecont)
self.assertTrue(os.path.exists(pyfilename))
......@@ -374,7 +380,7 @@ def one_plus_one():
return 1+1
'''
pyfilename = os.path.join(self.tmpDir, 'opo.py')
pyfilename = os.path.join(self.tmpDir, 'opo.py') # not ops.py
with open(pyfilename, 'w') as pyfile:
pyfile.write(filecont)
utsp = importlib.import_module('opo')
......@@ -382,10 +388,10 @@ def one_plus_one():
with open(pyfilename, 'r') as pyfile:
res, error_methods = srccoms_extract(pyfile, [], methods)
self.assertTrue(res)
self.assertTrue(
os.path.exists("samplecode_temp/"
"one_plus_one_example.py"))
os.remove("samplecode_temp/" "one_plus_one_example.py")
expectedFile = os.path.join("samplecode_temp",
"one_plus_one_example.py")
self.assertTrue(os.path.exists(expectedFile))
os.remove(expectedFile)
def test_with_empty_wlist(self):
"""
......@@ -429,11 +435,12 @@ def three_plus_three():
res, error_methods = srccoms_extract(pyfile, ['three_plus_three'],
methods)
self.assertTrue(res)
self.assertTrue(
os.path.exists("samplecode_temp/four_plus_four_example.py"))
os.remove("samplecode_temp/" "four_plus_four_example.py")
self.assertFalse(
os.path.exists("samplecode_temp/three_plus_three_example.py"))
expectedFile = os.path.join("samplecode_temp",
"four_plus_four_example.py")
self.assertTrue(os.path.exists(expectedFile))
os.remove(expectedFile)
self.assertFalse(os.path.exists(expectedFile))
# https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/fluid/layers/ops.py
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或