diff --git a/tools/print_signatures.py b/tools/print_signatures.py index a18774a8b57b6424e0d89188c537a2086f5aa183..5b250eaf8f98f4ccd7e1cac206b85ce8b2a754be 100644 --- a/tools/print_signatures.py +++ b/tools/print_signatures.py @@ -27,19 +27,12 @@ import pydoc import hashlib import six import functools -import logging +import paddle member_dict = collections.OrderedDict() visited_modules = set() -# APIs that should not be printed into API.spec -omitted_list = [ - "paddle.fluid.LoDTensor.set", # Do not know why it should be omitted - "paddle.fluid.io.ComposeNotAligned", - "paddle.fluid.io.ComposeNotAligned.__init__", -] - def md5(doc): hash = hashlib.md5() @@ -74,13 +67,28 @@ def format_spec(spec): def queue_dict(member, cur_name): - if cur_name in omitted_list: - return - - doc_md5 = md5(member.__doc__) - - if inspect.isclass(member): + if cur_name != 'paddle': + try: + eval(cur_name) + except (AttributeError, NameError, SyntaxError) as e: + print( + "Error({}) occurred when `eval({})`, discard it.".format( + str(e), cur_name), + file=sys.stderr) + return + + if (inspect.isclass(member) or inspect.isfunction(member) or + inspect.ismethod(member)) and hasattr( + member, '__module__') and hasattr(member, '__name__'): args = member.__module__ + "." + member.__name__ + try: + eval(args) + except (AttributeError, NameError, SyntaxError) as e: + print( + "Error({}) occurred when `eval({})`, discard it for {}.".format( + str(e), args, cur_name), + file=sys.stderr) + return else: try: args = inspect.getargspec(member) @@ -95,6 +103,7 @@ def queue_dict(member, cur_name): if not has_type_error: args = format_spec(args) + doc_md5 = md5(member.__doc__) member_dict[cur_name] = "({}, ('document', '{}'))".format(args, doc_md5) @@ -106,8 +115,7 @@ def visit_member(parent_name, member, member_name=None): if inspect.isclass(member): queue_dict(member, cur_name) for name, value in inspect.getmembers(member): - if hasattr(value, '__name__') and (not name.startswith("_") or - name == "__init__"): + if hasattr(value, '__name__') and not name.startswith("_"): visit_member(cur_name, value) elif inspect.ismethoddescriptor(member): return @@ -149,11 +157,14 @@ def visit_all_module(mod): return visited_modules.add(mod) - - for member_name in ( - name - for name in (mod.__all__ if hasattr(mod, "__all__") else dir(mod)) - if not name.startswith("_")): + if hasattr(mod, "__all__"): + member_names = (name for name in mod.__all__ + if not name.startswith("_")) + elif mod_name == 'paddle': + member_names = dir(mod) + else: + return + for member_name in member_names: instance = getattr(mod, member_name, None) if instance is None: continue @@ -168,17 +179,19 @@ def visit_all_module(mod): visit_all_module(instance) else: if member_name != instance.__name__: - logging.warn( + print( "Found alias API, alias name is: {}, original name is: {}". - format(member_name, instance.__name__)) + format(member_name, instance.__name__), + file=sys.stderr) visit_member(mod.__name__, instance, member_name) else: visit_member(mod.__name__, instance) -modules = sys.argv[1].split(",") -for m in modules: - visit_all_module(importlib.import_module(m)) +if __name__ == '__main__': + modules = sys.argv[1].split(",") + for m in modules: + visit_all_module(importlib.import_module(m)) -for name in member_dict: - print(name, member_dict[name]) + for name in member_dict: + print(name, member_dict[name]) diff --git a/tools/sampcd_processor.py b/tools/sampcd_processor.py index a8a717da027e074b6b817e5e84139df3bd65746c..cd5d9b5c08ad7afcf442d106f7cf7c058cba8e68 100644 --- a/tools/sampcd_processor.py +++ b/tools/sampcd_processor.py @@ -19,8 +19,8 @@ import multiprocessing import math import platform import inspect -import paddle -import paddle.fluid +#import paddle +#import paddle.fluid import json import argparse import shutil @@ -44,9 +44,7 @@ if logger.handlers: else: console = logging.StreamHandler() logger.addHandler(console) -console.setFormatter( - logging.Formatter( - "%(asctime)s - %(funcName)s:%(lineno)d - %(levelname)s - %(message)s")) +console.setFormatter(logging.Formatter("%(message)s")) RUN_ON_DEVICE = 'cpu' GPU_ID = 0 @@ -125,29 +123,41 @@ def sampcd_extract_and_run(srccom, name, htype="def", hname=""): name(str): the name of the API. msg(str): messages """ - global GPU_ID, RUN_ON_DEVICE, SAMPLECODE_TEMPDIR + sample_code_filenames = sampcd_extract_to_file(srccom, name, htype, hname) + if not sample_code_filenames: + return False, name, 'No sample code!' + + results = [] + msgs = [] + for tfname in sample_code_filenames: + result, msg = execute_samplecode_test(tfname) + results.append(result) + msgs.append(msg) + + if not all(results): + failed_fn = [] + for i, result in enumerate(results): + if not result: + failed_fn.append(sample_code_filenames[i]) + return False, name, 'failed sample codes: ' + ','.join(failed_fn) + return True, name, 'success!' + + +def sampcd_extract_to_file(srccom, name, htype="def", hname=""): + """ + Extract sample codes from __doc__, and write them to files. - result = True - msg = None + Args: + srccom(str): the source comment of some API whose + example codes will be extracted and run. + name(str): the name of the API. + htype(str): the type of hint banners, def/class/method. + hname(str): the name of the hint banners , e.t. def hname. - def sampcd_header_print(name, sampcd, htype, hname): - """ - print hint banner headers. - - Args: - name(str): the name of the API. - sampcd(str): sample code string - htype(str): the type of hint banners, def/class/method. - hname(str): the name of the hint banners , e.t. def hname. - flushed. - """ - print(htype, " name:", hname) - print("-----------------------") - print("Sample code ", str(y), " extracted for ", name, " :") - print(sampcd) - print("----example code check----\n") - print("executing sample code .....") - print("execution result:") + Returns: + sample_code_filenames(list of str) + """ + global GPU_ID, RUN_ON_DEVICE, SAMPLECODE_TEMPDIR sampcd_begins = find_all(srccom, " code-block:: python") if len(sampcd_begins) == 0: @@ -161,11 +171,11 @@ def sampcd_extract_and_run(srccom, name, htype="def", hname=""): "Deprecated sample code style:\n\n Examples:\n\n >>>codeline\n >>>codeline\n\n\n ", "Please use '.. code-block:: python' to ", "format sample code.\n") - result = False + return [] else: print("Error: No sample code!\n") - result = False - + return [] + sample_code_filenames = [] for y in range(1, len(sampcd_begins) + 1): sampcd_begin = sampcd_begins[y - 1] sampcd = srccom[sampcd_begin + len(" code-block:: python") + 1:] @@ -200,37 +210,50 @@ def sampcd_extract_and_run(srccom, name, htype="def", hname=""): tfname = os.path.join(SAMPLECODE_TEMPDIR, '{}_example{}'.format( name, '.py' if len(sampcd_begins) == 1 else '_{}.py'.format(y))) - logging.info('running %s', tfname) with open(tfname, 'w') as tempf: tempf.write(sampcd) - if platform.python_version()[0] == "2": - cmd = ["python", tfname] - elif platform.python_version()[0] == "3": - cmd = ["python3", tfname] - else: - print("Error: fail to parse python version!") - result = False - exit(1) + sample_code_filenames.append(tfname) + return sample_code_filenames - subprc = subprocess.Popen( - cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - output, error = subprc.communicate() - msg = "".join(output.decode(encoding='utf-8')) - err = "".join(error.decode(encoding='utf-8')) - if subprc.returncode != 0: - print("\nSample code error found in ", name, ":\n") - sampcd_header_print(name, sampcd, htype, hname) - print("subprocess return code: ", str(subprc.returncode)) - print("Error Raised from Sample Code ", name, " :\n") - print(err) - print(msg) - logging.warning('%s error: %s', tfname, err) - logging.warning('%s msg: %s', tfname, msg) - result = False - # msg is the returned code execution report +def execute_samplecode_test(tfname): + result = True + msg = None + if platform.python_version()[0] in ["2", "3"]: + cmd = [sys.executable, tfname] + else: + print("Error: fail to parse python version!") + result = False + exit(1) + + logging.info('running %s', tfname) + print("\n----example code check----") + print("executing sample code .....", tfname) + subprc = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output, error = subprc.communicate() + msg = "".join(output.decode(encoding='utf-8')) + err = "".join(error.decode(encoding='utf-8')) + + if subprc.returncode != 0: + print("Sample code error found in ", tfname, ":") + print("-----------------------") + print(open(tfname).read()) + print("-----------------------") + print("subprocess return code: ", str(subprc.returncode)) + print("Error Raised from Sample Code ", tfname, " :") + print(err) + print(msg) + print("----example code check failed----\n") + logging.warning('%s error: %s', tfname, err) + logging.warning('%s msg: %s', tfname, msg) + result = False + else: + print("----example code check success----\n") - return result, name, msg + # msg is the returned code execution report + + return result, tfname, msg def single_defcom_extract(start_from, srcls, is_class_begin=False): @@ -326,7 +349,7 @@ def srccoms_extract(srcfile, wlist, methods): if allidx != -1: alllist = [] # get all list for layers/ops.py - if srcfile.name.find("ops.py") != -1: + if srcfile.name.find("fluid/layers/ops.py") != -1: for ai in range(0, len(srcls)): if srcls[ai].startswith("__all__"): lb = srcls[ai].find('[') @@ -352,7 +375,7 @@ def srccoms_extract(srcfile, wlist, methods): api_count = 0 handled = [] # get src contents in layers/ops.py - if srcfile.name.find("ops.py") != -1: + if srcfile.name.find("fluid/layers/ops.py") != -1: for i in range(0, len(srcls)): opname = None opres = re.match(r"^(\w+)\.__doc__", srcls[i]) @@ -536,25 +559,24 @@ def run_a_test(tc_filename): def get_filenames(): ''' - this function will get the modules that pending for check. + this function will get the sample code files that pending for check. Returns: - list: the modules pending for check . + dict: the sample code files pending for check . ''' - filenames = [] global methods # write global whl_error - methods = [] + import paddle whl_error = [] get_incrementapi() - API_spec = API_DIFF_SPEC_FN - with open(API_spec) as f: + all_sample_code_filenames = {} + with open(API_DIFF_SPEC_FN) as f: for line in f.readlines(): api = line.replace('\n', '') try: - module = eval(api).__module__ + api_obj = eval(api) except AttributeError: whl_error.append(api) continue @@ -562,47 +584,12 @@ def get_filenames(): logger.warning('line:%s, api:%s', line, api) # paddle.Tensor. continue - if len(module.split('.')) > 1: - filename = '../python/' - # work for .so? - module_py = '%s.py' % module.split('.')[-1] - for i in range(0, len(module.split('.')) - 1): - filename = filename + '%s/' % module.split('.')[i] - filename = filename + module_py - else: - filename = '' - logger.warning("WARNING: Exception in getting api:%s module:%s", - api, module) - if filename in filenames: - continue - elif not filename: - logger.warning('filename invalid: %s', line) - continue - elif not os.path.exists(filename): - logger.warning('file not exists: %s', filename) - continue - else: - filenames.append(filename) - # get all methods - method = '' - if inspect.isclass(eval(api)): - name = api.split('.')[-1] - elif inspect.isfunction(eval(api)): - name = api.split('.')[-1] - elif inspect.ismethod(eval(api)): - name = '%s.%s' % (api.split('.')[-2], api.split('.')[-1]) - else: - name = '' - logger.warning( - "WARNING: Exception when getting api:%s, line:%s", api, - line) - for j in range(2, len(module.split('.'))): - method = method + '%s.' % module.split('.')[j] - method = method + name - if method not in methods: - methods.append(method) - os.remove(API_spec) - return filenames + if hasattr(api_obj, '__doc__') and api_obj.__doc__: + sample_code_filenames = sampcd_extract_to_file(api_obj.__doc__, + api) + for tfname in sample_code_filenames: + all_sample_code_filenames[tfname] = api + return all_sample_code_filenames def get_api_md5(path): @@ -744,14 +731,6 @@ if __name__ == '__main__': if len(filenames) == 0 and len(whl_error) == 0: logger.info("-----API_PR.spec is the same as API_DEV.spec-----") exit(0) - rm_file = [] - for f in filenames: - for w_file in wlist_file: - if f.startswith(w_file): - rm_file.append(f) - filenames.remove(f) - if len(rm_file) != 0: - logger.info("REMOVE white files: %s", rm_file) logger.info("API_PR is diff from API_DEV: %s", filenames) threads = multiprocessing.cpu_count() @@ -759,7 +738,7 @@ if __name__ == '__main__': threads = args.threads po = multiprocessing.Pool(threads) # results = po.map_async(test, divided_file_list) - results = po.map_async(run_a_test, filenames) + results = po.map_async(execute_samplecode_test, filenames.keys()) po.close() po.join() diff --git a/tools/test_sampcd_processor.py b/tools/test_sampcd_processor.py index f8bcb662e879b959fcb4247a4b350d9a18e4a3d0..6b675d2a11508c248819f999e8e019792489f24f 100644 --- a/tools/test_sampcd_processor.py +++ b/tools/test_sampcd_processor.py @@ -271,6 +271,10 @@ class Test_get_wlist(unittest.TestCase): class Test_srccoms_extract(unittest.TestCase): def setUp(self): self.tmpDir = tempfile.mkdtemp() + print('tmpDir=', self.tmpDir) + self.opsDir = os.path.join(self.tmpDir, 'fluid/layers') + os.makedirs(self.opsDir) + sys.path.append(self.opsDir) sys.path.append(self.tmpDir) self.api_pr_spec_filename = os.path.abspath( os.path.join(os.getcwd(), "..", 'paddle/fluid/API_PR.spec')) @@ -283,7 +287,7 @@ class Test_srccoms_extract(unittest.TestCase): ])) def tearDown(self): - sys.path.remove(self.tmpDir) + #sys.path.remove(self.tmpDir) shutil.rmtree(self.tmpDir) os.remove(self.api_pr_spec_filename) @@ -305,9 +309,11 @@ def exp(): add_sample_code(globals()["exp"], r""" Examples: .. code-block:: python - import paddle - x = paddle.to_tensor([-0.4, -0.2, 0.1, 0.3]) - out = paddle.exp(x) + + # import paddle + # x = paddle.to_tensor([-0.4, -0.2, 0.1, 0.3]) + # out = paddle.exp(x) + out = [0.67032005, 0.81873075, 1.10517092, 1.34985881] print(out) # [0.67032005 0.81873075 1.10517092 1.34985881] """) @@ -332,7 +338,7 @@ add_sample_code(globals()["two_plus_two"], """ print(2+2) """) ''' - pyfilename = os.path.join(self.tmpDir, 'ops.py') + pyfilename = os.path.join(self.opsDir, 'ops.py') with open(pyfilename, 'w') as pyfile: pyfile.write(filecont) self.assertTrue(os.path.exists(pyfilename)) @@ -374,7 +380,7 @@ def one_plus_one(): return 1+1 ''' - pyfilename = os.path.join(self.tmpDir, 'opo.py') + pyfilename = os.path.join(self.tmpDir, 'opo.py') # not ops.py with open(pyfilename, 'w') as pyfile: pyfile.write(filecont) utsp = importlib.import_module('opo') @@ -382,10 +388,10 @@ def one_plus_one(): with open(pyfilename, 'r') as pyfile: res, error_methods = srccoms_extract(pyfile, [], methods) self.assertTrue(res) - self.assertTrue( - os.path.exists("samplecode_temp/" - "one_plus_one_example.py")) - os.remove("samplecode_temp/" "one_plus_one_example.py") + expectedFile = os.path.join("samplecode_temp", + "one_plus_one_example.py") + self.assertTrue(os.path.exists(expectedFile)) + os.remove(expectedFile) def test_with_empty_wlist(self): """ @@ -429,11 +435,12 @@ def three_plus_three(): res, error_methods = srccoms_extract(pyfile, ['three_plus_three'], methods) self.assertTrue(res) - self.assertTrue( - os.path.exists("samplecode_temp/four_plus_four_example.py")) - os.remove("samplecode_temp/" "four_plus_four_example.py") - self.assertFalse( - os.path.exists("samplecode_temp/three_plus_three_example.py")) + + expectedFile = os.path.join("samplecode_temp", + "four_plus_four_example.py") + self.assertTrue(os.path.exists(expectedFile)) + os.remove(expectedFile) + self.assertFalse(os.path.exists(expectedFile)) # https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/fluid/layers/ops.py