From 2328921f19fd7983b69a1952de23fb7078e6074a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=20Wei=20=28=E4=BB=BB=E5=8D=AB=29?= Date: Sun, 25 Apr 2021 18:43:22 +0800 Subject: [PATCH] Cleanup the unused codes for samplecode testing (#32525) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * update testcases * remove unused codes * update the docstring for samcd_processor * no need to import the six module * 我也不知道为何有一个前导空格，但现在有单元测试，取消这个空格没啥问题 * add unittests for print_signatures; add the first case for 'required' mechanism when executing sample code testing * there is no paddle installed in PR-CI-APPROVAL test=document_fix --- tools/check_file_diff_approvals.sh | 1 + tools/print_signatures.py | 6 +- tools/sampcd_processor.py | 384 +++-------------------------- tools/test_print_signatures.py | 95 +++++++ tools/test_sampcd_processor.py | 330 ++++++------------------- 5 files changed, 206 insertions(+), 610 deletions(-) create mode 100644 tools/test_print_signatures.py diff --git a/tools/check_file_diff_approvals.sh b/tools/check_file_diff_approvals.sh index 05466883e58..b1395c28878 100644 --- a/tools/check_file_diff_approvals.sh +++ b/tools/check_file_diff_approvals.sh @@ -84,6 +84,7 @@ function run_test_sampcd_processor() { CUR_PWD=$(pwd) cd ${PADDLE_ROOT}/tools python test_sampcd_processor.py + python test_print_signatures.py cd ${CUR_PWD} } diff --git a/tools/print_signatures.py b/tools/print_signatures.py index 5b250eaf8f9..cfe34fa3426 100644 --- a/tools/print_signatures.py +++ b/tools/print_signatures.py @@ -25,9 +25,8 @@ import collections import sys import pydoc import hashlib -import six +import platform import functools -import paddle member_dict = collections.OrderedDict() @@ -131,7 +130,7 @@ def visit_member(parent_name, member, member_name=None): def is_primitive(instance): - int_types = (int, long) if six.PY2 else (int, ) + int_types = (int, long) if platform.python_version()[0] == "2" else (int, ) pritimitive_types = int_types + (float, str) if isinstance(instance, pritimitive_types): return True @@ -189,6 +188,7 @@ def visit_all_module(mod): if __name__ == '__main__': + import paddle modules = sys.argv[1].split(",") for m in modules: visit_all_module(importlib.import_module(m)) diff --git a/tools/sampcd_processor.py b/tools/sampcd_processor.py index cd5d9b5c08a..52777cd59ba 100644 --- a/tools/sampcd_processor.py +++ b/tools/sampcd_processor.py @@ -19,8 +19,6 @@ import multiprocessing import math import platform import inspect -#import paddle -#import paddle.fluid import json import argparse import shutil @@ -28,8 +26,8 @@ import re import logging """ please make sure to run in the tools path -usage: python sample_test.py {arg1} -arg1: the first arg defined running in gpu version or cpu version +usage: python sample_test.py {cpu or gpu} + {cpu or gpu}: running in cpu version or gpu version for example, you can run cpu version python2 testing like this: @@ -105,44 +103,6 @@ def check_indent(cdline): return indent -# srccom: raw comments in the source,including ''' and original indent -def sampcd_extract_and_run(srccom, name, htype="def", hname=""): - """ - Extract and run sample codes from source comment and - the result will be returned. - - Args: - srccom(str): the source comment of some API whose - example codes will be extracted and run. - name(str): the name of the API. - htype(str): the type of hint banners, def/class/method. - hname(str): the name of the hint banners , e.t. def hname. - - Returns: - result: True or False - name(str): the name of the API. - msg(str): messages - """ - sample_code_filenames = sampcd_extract_to_file(srccom, name, htype, hname) - if not sample_code_filenames: - return False, name, 'No sample code!' - - results = [] - msgs = [] - for tfname in sample_code_filenames: - result, msg = execute_samplecode_test(tfname) - results.append(result) - msgs.append(msg) - - if not all(results): - failed_fn = [] - for i, result in enumerate(results): - if not result: - failed_fn.append(sample_code_filenames[i]) - return False, name, 'failed sample codes: ' + ','.join(failed_fn) - return True, name, 'success!' - - def sampcd_extract_to_file(srccom, name, htype="def", hname=""): """ Extract sample codes from __doc__, and write them to files. @@ -158,8 +118,9 @@ def sampcd_extract_to_file(srccom, name, htype="def", hname=""): sample_code_filenames(list of str) """ global GPU_ID, RUN_ON_DEVICE, SAMPLECODE_TEMPDIR + CODE_BLOCK_INTERDUCTORY = "code-block:: python" - sampcd_begins = find_all(srccom, " code-block:: python") + sampcd_begins = find_all(srccom, CODE_BLOCK_INTERDUCTORY) if len(sampcd_begins) == 0: # detect sample codes using >>> to format and consider this situation as wrong print(htype, " name:", hname) @@ -178,7 +139,7 @@ def sampcd_extract_to_file(srccom, name, htype="def", hname=""): sample_code_filenames = [] for y in range(1, len(sampcd_begins) + 1): sampcd_begin = sampcd_begins[y - 1] - sampcd = srccom[sampcd_begin + len(" code-block:: python") + 1:] + sampcd = srccom[sampcd_begin + len(CODE_BLOCK_INTERDUCTORY) + 1:] sampcd = sampcd.split("\n") # remove starting empty lines while sampcd[0].replace(' ', '').replace('\t', '') == '': @@ -216,7 +177,18 @@ def sampcd_extract_to_file(srccom, name, htype="def", hname=""): return sample_code_filenames -def execute_samplecode_test(tfname): +def execute_samplecode(tfname): + """ + Execute a sample-code test. + + Args: + tfname: the filename of the samplecode. + + Returns: + result: success or not + tfname: same as the input argument + msg: the stdout output of the samplecode executing. + """ result = True msg = None if platform.python_version()[0] in ["2", "3"]: @@ -226,6 +198,14 @@ def execute_samplecode_test(tfname): result = False exit(1) + # check required envisonment + with open(tfname, 'r') as f: + for line in f.readlines(): + if re.match(r'#\s*required\s*:\s*(distributed|gpu|skip)', line): + result = True + return result, tfname, '{} is skipped. cause: {}'.format(tfname, + line) + logging.info('running %s', tfname) print("\n----example code check----") print("executing sample code .....", tfname) @@ -252,311 +232,9 @@ def execute_samplecode_test(tfname): print("----example code check success----\n") # msg is the returned code execution report - return result, tfname, msg -def single_defcom_extract(start_from, srcls, is_class_begin=False): - """ - to extract a def function/class/method comments body - - Args: - start_from(int): the line num of "def" header - srcls(list): the source file in lines - is_class_begin(bool): whether the start_from is a beginning a class. \ - For a sole class body itself may end up with its method if it has no - docstring. But the body of \ - a common def function can only be ended up by a none-indented def/class - - Returns: - string : the extracted comment body, inclusive of its quote marks. - - """ - - i = start_from - fcombody = "" # def comment body - comstart = -1 # the starting line index of comment mark "'''" or """""" - # if it is not -1, it indicates the loop is in the comment body - comstyle = 0 # comment mark style ,comments quoted with ''' is coded as 1 - # comments quoted with """ is coded as 2 - for x in range(i + 1, len(srcls)): - if is_class_begin: - if srcls[x].replace('\t', ' ').startswith(' def '): - break - if srcls[x].startswith('def ') or srcls[x].startswith('class '): - break - else: - if comstart == -1: - s = srcls[x].replace(" ", '').replace("\t", - '').replace("\n", '') - if s.startswith("\"\"\"") or s.startswith("r\"\"\""): - comstart = x - comstyle = 2 - continue - if (comstyle == 2 and comstart != -1 and - srcls[x].replace(" ", '').replace("\t", '').replace( - "\n", '').startswith("\"\"\"")): - break - if comstart == -1: - s = srcls[x].replace(" ", '').replace("\t", - '').replace("\n", '') - if s.startswith("\'\'\'") or s.startswith("r\'\'\'"): - comstart = x - comstyle = 1 - continue - if (comstyle == 1 and comstart != -1 and - srcls[x].replace(" ", '').replace("\t", '').replace( - "\n", '').startswith("\'\'\'")): - break - if (comstart != - -1): # when the comments start, begin to add line to fcombody - fcombody += srcls[x] - return fcombody - - -def srccoms_extract(srcfile, wlist, methods): - """ - Given a source file ``srcfile``, this function will - extract its API(doc comments) and run sample codes in the - API. - - Args: - srcfile(file): the source file - wlist(list): white list - methods(list): only elements of this list considered. - - Returns: - result: True or False - error_methods: the methods that failed. - """ - - process_result = True - error_methods = [] - srcc = srcfile.read() - # 2. get defs and classes header line number - # set file pointer to its beginning - srcfile.seek(0, 0) - srcls = srcfile.readlines() # source lines - - # 1. fetch__all__ list - allidx = srcc.find("__all__") - logger.debug('processing %s, methods: %s', srcfile.name, str(methods)) - srcfile_new, _ = os.path.splitext(srcfile.name) - srcfile_list = srcfile_new.split('/') - srcfile_str = '' - for i in range(4, len(srcfile_list)): - srcfile_str = srcfile_str + srcfile_list[i] + '.' - if allidx != -1: - alllist = [] - # get all list for layers/ops.py - if srcfile.name.find("fluid/layers/ops.py") != -1: - for ai in range(0, len(srcls)): - if srcls[ai].startswith("__all__"): - lb = srcls[ai].find('[') - rb = srcls[ai].find(']') - if lb == -1: - continue - allele = srcls[ai][lb + 1:rb].replace("'", '').replace( - " ", '').replace("\"", '') - alllist.append(allele) - if '' in alllist: - alllist.remove('') - else: - alllist_b = allidx + len("__all__") - allstr = srcc[alllist_b + srcc[alllist_b:].find("[") + 1:alllist_b + - srcc[alllist_b:].find("]")] - allstr = allstr.replace("\n", '').replace(" ", '').replace( - "'", '').replace("\"", '') - alllist = allstr.split(',') - if '' in alllist: - alllist.remove('') - api_alllist_count = len(alllist) - logger.debug('found %d items: %s', api_alllist_count, str(alllist)) - api_count = 0 - handled = [] - # get src contents in layers/ops.py - if srcfile.name.find("fluid/layers/ops.py") != -1: - for i in range(0, len(srcls)): - opname = None - opres = re.match(r"^(\w+)\.__doc__", srcls[i]) - if opres is not None: - opname = opres.group(1) - else: - opres = re.match( - r"^add_sample_code\(globals\(\)\[\"(\w+)\"\]", srcls[i]) - if opres is not None: - opname = opres.group(1) - if opname is not None: - if opname in wlist: - logger.info('%s is in the whitelist, skip it.', opname) - continue - else: - logger.debug('%s\'s docstring found.', opname) - comstart = i - for j in range(i, len(srcls)): - if srcls[j].find("\"\"\"") != -1: - comstart = i - opcom = "" - for j in range(comstart + 1, len(srcls)): - opcom += srcls[j] - if srcls[j].find("\"\"\"") != -1: - break - result, _, _ = sampcd_extract_and_run(opcom, opname, "def", - opname) - if not result: - error_methods.append(opname) - process_result = False - api_count += 1 - handled.append( - opname) # ops.py also has normal formatted functions - # use list 'handled' to mark the functions have been handled here - # which will be ignored in the following step - # handled what? - logger.debug('%s already handled.', str(handled)) - for i in range(0, len(srcls)): - if srcls[i].startswith( - 'def '): # a function header is detected in line i - f_header = srcls[i].replace(" ", '') - fn = f_header[len('def'):f_header.find('(')] # function name - if "%s%s" % (srcfile_str, fn) not in methods: - logger.info( - '[file:%s, function:%s] not in methods list, skip it.', - srcfile_str, fn) - continue - if fn in handled: - continue - if fn in alllist: - api_count += 1 - if fn in wlist or fn + "@" + srcfile.name in wlist: - logger.info('[file:%s, function:%s] skip by wlist.', - srcfile_str, fn) - continue - fcombody = single_defcom_extract(i, srcls) - if fcombody == "": # if no comment - print("def name:", fn) - print("-----------------------") - print("WARNING: no comments in function ", fn, - ", but it deserves.") - continue - else: - result, _, _ = sampcd_extract_and_run(fcombody, fn, - "def", fn) - if not result: - error_methods.append(fn) - process_result = False - - if srcls[i].startswith('class '): - c_header = srcls[i].replace(" ", '') - cn = c_header[len('class'):c_header.find('(')] # class name - if '%s%s' % (srcfile_str, cn) not in methods: - logger.info( - '[file:%s, class:%s] not in methods list, skip it.', - srcfile_str, cn) - continue - if cn in handled: - continue - if cn in alllist: - api_count += 1 - if cn in wlist or cn + "@" + srcfile.name in wlist: - logger.info('[file:%s, class:%s] skip by wlist.', - srcfile_str, cn) - continue - # class comment - classcom = single_defcom_extract(i, srcls, True) - if classcom != "": - result, _, _ = sampcd_extract_and_run(classcom, cn, - "class", cn) - if not result: - error_methods.append(cn) - process_result = False - else: - print("WARNING: no comments in class itself ", cn, - ", but it deserves.\n") - # handling methods in class bodies - for x in range( - i + 1, - len(srcls)): # from the next line of class header - if (srcls[x].startswith('def ') or - srcls[x].startswith('class ')): - break - else: - # member method def header - srcls[x] = srcls[x].replace('\t', ' ') - if (srcls[x].startswith( - ' def ')): # detect a mehtod header.. - thisl = srcls[x] - indent = len(thisl) - len(thisl.lstrip()) - mn = thisl[indent + len('def '):thisl.find( - '(')] # method name - name = cn + "." + mn # full name - if '%s%s' % ( - srcfile_str, name - ) not in methods: # class method not in api.spec - logger.info( - '[file:%s, func:%s] not in methods, skip it.', - srcfile_str, name) - continue - if mn.startswith('_'): - logger.info( - '[file:%s, func:%s] startswith _, it\'s private method, skip it.', - srcfile_str, name) - continue - if name in wlist or name + "@" + srcfile.name in wlist: - logger.info( - '[file:%s, class:%s] skip by wlist.', - srcfile_str, name) - continue - thismethod = [thisl[indent:] - ] # method body lines - # get all the lines of a single method body - # into thismethod(list) - # and send it to single_defcom_extract - for y in range(x + 1, len(srcls)): - srcls[y] = srcls[y].replace('\t', ' ') - if (srcls[y].startswith('def ') or - srcls[y].startswith('class ')): - # end of method - break - elif srcls[y].startswith(' def '): - # end of method - break - else: - thismethod.append(srcls[y][indent:]) - thismtdcom = single_defcom_extract(0, - thismethod) - if thismtdcom != "": - result, _, _ = sampcd_extract_and_run( - thismtdcom, name, "method", name) - if not result: - error_methods.append(name) - process_result = False - else: - logger.warning('__all__ not found in file:%s', srcfile.name) - - return process_result, error_methods - - -def test(file_list): - global methods # readonly - process_result = True - for file in file_list: - with open(file, 'r') as src: - if not srccoms_extract(src, wlist, methods): - process_result = False - return process_result - - -def run_a_test(tc_filename): - """ - execute a sample code-block. - """ - global methods # readonly - process_result = True - with open(tc_filename, 'r') as src: - process_result, error_methods = srccoms_extract(src, wlist, methods) - return process_result, tc_filename, error_methods - - def get_filenames(): ''' this function will get the sample code files that pending for check. @@ -593,6 +271,15 @@ def get_filenames(): def get_api_md5(path): + """ + read the api spec file, and scratch the md5sum value of every api's docstring. + + Args: + path: the api spec file. ATTENTION the path relative + + Returns: + api_md5(dict): key is the api's real fullname, value is the md5sum. + """ api_md5 = {} API_spec = '%s/%s' % (os.path.abspath(os.path.join(os.getcwd(), "..")), path) @@ -737,8 +424,7 @@ if __name__ == '__main__': if args.threads: threads = args.threads po = multiprocessing.Pool(threads) - # results = po.map_async(test, divided_file_list) - results = po.map_async(execute_samplecode_test, filenames.keys()) + results = po.map_async(execute_samplecode, filenames.keys()) po.close() po.join() diff --git a/tools/test_print_signatures.py b/tools/test_print_signatures.py new file mode 100644 index 00000000000..7cbdbb56cb1 --- /dev/null +++ b/tools/test_print_signatures.py @@ -0,0 +1,95 @@ +#! /usr/bin/env python + +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +TestCases for print_signatures.py + +sample lines from API_DEV.spec: + paddle.autograd.backward (ArgSpec(args=['tensors', 'grad_tensors', 'retain_graph'], varargs=None, keywords=None, defaults=(None, False)), ('document', '33a4434c9d123331499334fbe0274870')) + paddle.autograd.PyLayer (paddle.autograd.py_layer.PyLayer, ('document', 'c26adbbf5f1eb43d16d4a399242c979e')) + paddle.autograd.PyLayer.apply (ArgSpec(args=['cls'], varargs=args, keywords=kwargs, defaults=None), ('document', 'cb78696dc032fb8af2cba8504153154d')) +""" +import unittest +import hashlib +import inspect +import functools +from print_signatures import md5 +from print_signatures import get_functools_partial_spec +from print_signatures import format_spec +from print_signatures import queue_dict +from print_signatures import member_dict + + +def func_example(param_a, param_b): + """ + example function + """ + pass + + +def func_example_2(func=functools.partial(func_example, 1)): + """ + example function 2 + """ + pass + + +class ClassExample(): + """ + example Class + """ + + def example_method(self): + """ + class method + """ + pass + + +class Test_all_in_print_signatures(unittest.TestCase): + def test_md5(self): + algo = hashlib.md5() + algo.update(func_example.__doc__.encode('utf-8')) + digest = algo.hexdigest() + self.assertEqual(digest, md5(func_example.__doc__)) + + def test_get_functools_partial_spec(self): + partailed_func = functools.partial(func_example, 1) + # args = inspect.getargspec(partailed_func) + self.assertEqual('func_example(args=(1,), keywords={})', + get_functools_partial_spec(partailed_func)) + + +class Test_format_spec(unittest.TestCase): + def test_normal_func_spec(self): + args = inspect.getargspec(func_example) + self.assertEqual( + '''ArgSpec(args=['param_a', 'param_b'], varargs=None, keywords=None, defaults=None)''', + format_spec(args)) + + def test_func_spec_with_partialedfunc_as_param_default(self): + # but there is no function belongs to this type in API_DEV.spec + args = inspect.getargspec(func_example_2) + self.assertEqual( + '''ArgSpec(args=['func'], varargs=None, keywords=None, defaults=('func_example(args=(1,), keywords={})',))''', + format_spec(args)) + + +class Test_queue_dict(unittest.TestCase): + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/test_sampcd_processor.py b/tools/test_sampcd_processor.py index 6b675d2a115..7836728247f 100644 --- a/tools/test_sampcd_processor.py +++ b/tools/test_sampcd_processor.py @@ -22,12 +22,13 @@ import sys import importlib from sampcd_processor import find_all from sampcd_processor import check_indent -from sampcd_processor import sampcd_extract_and_run -from sampcd_processor import single_defcom_extract -from sampcd_processor import srccoms_extract from sampcd_processor import get_api_md5 from sampcd_processor import get_incrementapi from sampcd_processor import get_wlist +from sampcd_processor import sampcd_extract_to_file +from sampcd_processor import execute_samplecode + +SAMPLECODE_TEMP_DIR = 'samplecode_temp' class Test_find_all(unittest.TestCase): @@ -53,107 +54,95 @@ class Test_check_indent(unittest.TestCase): self.assertEqual(4, check_indent("\thello paddle")) -class Test_sampcd_extract_and_run(unittest.TestCase): +class Test_execute_samplecode(unittest.TestCase): + def setUp(self): + if not os.path.exists(SAMPLECODE_TEMP_DIR): + os.mkdir(SAMPLECODE_TEMP_DIR) + self.successSampleCodeFile = os.path.join(SAMPLECODE_TEMP_DIR, + 'samplecode_success.py') + with open(self.successSampleCodeFile, 'w') as f: + f.write('print(1+1)') + self.failedSampleCodeFile = os.path.join(SAMPLECODE_TEMP_DIR, + 'samplecode_failed.py') + with open(self.failedSampleCodeFile, 'w') as f: + f.write('print(1/0)') + + def tearDown(self): + os.remove(self.successSampleCodeFile) + os.remove(self.failedSampleCodeFile) + + def test_run_success(self): + result, tfname, msg = execute_samplecode(self.successSampleCodeFile) + self.assertTrue(result) + self.assertEqual(self.successSampleCodeFile, tfname) + self.assertIsNotNone(msg) + self.assertLess(msg.find('skipped'), 0) + + def test_run_failed(self): + result, tfname, msg = execute_samplecode(self.failedSampleCodeFile) + self.assertFalse(result) + self.assertEqual(self.failedSampleCodeFile, tfname) + self.assertIsNotNone(msg) + self.assertLess(msg.find('skipped'), 0) + + def test_testcases_skipped(self): + ... + tfname = os.path.join(SAMPLECODE_TEMP_DIR, 'samplecode_skipped.py') + with open(tfname, 'w') as f: + f.write("# required: distributed\nprint(1/0)") + result, _, msg = execute_samplecode(tfname) + self.assertTrue(result) + self.assertGreaterEqual(msg.find('skipped'), 0) + os.remove(tfname) + + +class Test_sampcd_extract_to_file(unittest.TestCase): def setUp(self): - if not os.path.exists('samplecode_temp/'): - os.mkdir('samplecode_temp/') + if not os.path.exists(SAMPLECODE_TEMP_DIR): + os.mkdir(SAMPLECODE_TEMP_DIR) - def test_run_a_defs_samplecode(self): + def tearDown(self): + shutil.rmtree(SAMPLECODE_TEMP_DIR) + + def test_1_samplecode(self): comments = """ Examples: .. code-block:: python + print(1+1) """ funcname = 'one_plus_one' - res, name, msg = sampcd_extract_and_run(comments, funcname) - self.assertTrue(res) - self.assertEqual(funcname, name) + sample_code_filenames = sampcd_extract_to_file(comments, funcname) + self.assertCountEqual( + [os.path.join(SAMPLECODE_TEMP_DIR, funcname + '_example.py')], + sample_code_filenames) - def test_run_a_def_no_code(self): + def test_no_samplecode(self): comments = """ placeholder """ funcname = 'one_plus_one' - res, name, msg = sampcd_extract_and_run(comments, funcname) - self.assertFalse(res) - self.assertEqual(funcname, name) + sample_code_filenames = sampcd_extract_to_file(comments, funcname) + self.assertCountEqual([], sample_code_filenames) - def test_run_a_def_raise_expection(self): + def test_2_samplecodes(self): comments = """ placeholder Examples: .. code-block:: python - print(1/0) - """ - funcname = 'one_plus_one' - res, name, msg = sampcd_extract_and_run(comments, funcname) - self.assertFalse(res) - self.assertEqual(funcname, name) - -class Test_single_defcom_extract(unittest.TestCase): - def test_extract_from_func(self): - defstr = ''' -import os -def foo(): - """ - foo is a function. - """ - pass -def bar(): - pass -''' - comm = single_defcom_extract( - 2, defstr.splitlines(True), is_class_begin=False) - self.assertEqual(" foo is a function.\n", comm) - pass - - def test_extract_from_func_with_no_docstring(self): - defstr = ''' -import os -def bar(): - pass -''' - comm = single_defcom_extract( - 2, defstr.splitlines(True), is_class_begin=False) - self.assertEqual('', comm) - pass + print(1/0) - def test_extract_from_class(self): - defstr = r''' -import os -class Foo(): - """ - Foo is a class. - second line. - """ - pass - def bar(): - pass -def foo(): - pass -''' - comm = single_defcom_extract( - 2, defstr.splitlines(True), is_class_begin=True) - rcomm = """ Foo is a class. - second line. -""" - self.assertEqual(rcomm, comm) - pass + .. code-block:: python - def test_extract_from_class_with_no_docstring(self): - defstr = ''' -import os -class Foo(): - pass - def bar(): - pass -def foo(): - pass -''' - comm = single_defcom_extract( - 0, defstr.splitlines(True), is_class_begin=True) - self.assertEqual('', comm) + print(1+1) + """ + funcname = 'one_plus_one' + sample_code_filenames = sampcd_extract_to_file(comments, funcname) + self.assertCountEqual([ + os.path.join(SAMPLECODE_TEMP_DIR, funcname + '_example_1.py'), + os.path.join(SAMPLECODE_TEMP_DIR, funcname + '_example_2.py') + ], sample_code_filenames) class Test_get_api_md5(unittest.TestCase): @@ -268,181 +257,6 @@ class Test_get_wlist(unittest.TestCase): self.assertCountEqual(["deformable_conv"], gpu_not_white) -class Test_srccoms_extract(unittest.TestCase): - def setUp(self): - self.tmpDir = tempfile.mkdtemp() - print('tmpDir=', self.tmpDir) - self.opsDir = os.path.join(self.tmpDir, 'fluid/layers') - os.makedirs(self.opsDir) - sys.path.append(self.opsDir) - sys.path.append(self.tmpDir) - self.api_pr_spec_filename = os.path.abspath( - os.path.join(os.getcwd(), "..", 'paddle/fluid/API_PR.spec')) - with open(self.api_pr_spec_filename, 'w') as f: - f.write("\n".join([ - """one_plus_one (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', "one_plus_one"))""", - """two_plus_two (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', "two_plus_two"))""", - """three_plus_three (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', "three_plus_three"))""", - """four_plus_four (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', "four_plus_four"))""", - ])) - - def tearDown(self): - #sys.path.remove(self.tmpDir) - shutil.rmtree(self.tmpDir) - os.remove(self.api_pr_spec_filename) - - def test_from_ops_py(self): - filecont = ''' -def add_sample_code(obj, docstr): - pass - -__unary_func__ = [ - 'exp', -] - -__all__ = [] -__all__ += __unary_func__ -__all__ += ['one_plus_one'] - -def exp(): - pass -add_sample_code(globals()["exp"], r""" -Examples: - .. code-block:: python - - # import paddle - # x = paddle.to_tensor([-0.4, -0.2, 0.1, 0.3]) - # out = paddle.exp(x) - out = [0.67032005, 0.81873075, 1.10517092, 1.34985881] - print(out) - # [0.67032005 0.81873075 1.10517092 1.34985881] -""") - -def one_plus_one(): - return 1+1 - -one_plus_one.__doc__ = """ - placeholder - - Examples: - .. code-block:: python - print(1+1) -""" - -__all__ += ['two_plus_two'] -def two_plus_two(): - return 2+2 -add_sample_code(globals()["two_plus_two"], """ - Examples: - .. code-block:: python - print(2+2) -""") -''' - pyfilename = os.path.join(self.opsDir, 'ops.py') - with open(pyfilename, 'w') as pyfile: - pyfile.write(filecont) - self.assertTrue(os.path.exists(pyfilename)) - utsp = importlib.import_module('ops') - print('testing srccoms_extract from ops.py') - methods = ['one_plus_one', 'two_plus_two', 'exp'] - # os.remove("samplecode_temp/" "one_plus_one_example.py") - self.assertFalse( - os.path.exists("samplecode_temp/" - "one_plus_one_example.py")) - with open(pyfilename, 'r') as pyfile: - res, error_methods = srccoms_extract(pyfile, [], methods) - self.assertTrue(res) - self.assertTrue( - os.path.exists("samplecode_temp/" - "one_plus_one_example.py")) - os.remove("samplecode_temp/" "one_plus_one_example.py") - self.assertTrue( - os.path.exists("samplecode_temp/" - "two_plus_two_example.py")) - os.remove("samplecode_temp/" "two_plus_two_example.py") - self.assertTrue(os.path.exists("samplecode_temp/" "exp_example.py")) - os.remove("samplecode_temp/" "exp_example.py") - - def test_from_not_ops_py(self): - filecont = ''' -__all__ = [ - 'one_plus_one' -] - -def one_plus_one(): - """ - placeholder - - Examples: - .. code-block:: python - print(1+1) - """ - return 1+1 - -''' - pyfilename = os.path.join(self.tmpDir, 'opo.py') # not ops.py - with open(pyfilename, 'w') as pyfile: - pyfile.write(filecont) - utsp = importlib.import_module('opo') - methods = ['one_plus_one'] - with open(pyfilename, 'r') as pyfile: - res, error_methods = srccoms_extract(pyfile, [], methods) - self.assertTrue(res) - expectedFile = os.path.join("samplecode_temp", - "one_plus_one_example.py") - self.assertTrue(os.path.exists(expectedFile)) - os.remove(expectedFile) - - def test_with_empty_wlist(self): - """ - see test_from_ops_py - """ - pass - - def test_with_wlist(self): - filecont = ''' -__all__ = [ - 'four_plus_four', - 'three_plus_three' - ] - -def four_plus_four(): - """ - placeholder - - Examples: - .. code-block:: python - print(4+4) - """ - return 4+4 -def three_plus_three(): - """ - placeholder - - Examples: - .. code-block:: python - print(3+3) - """ - return 3+3 - -''' - pyfilename = os.path.join(self.tmpDir, 'three_and_four.py') - with open(pyfilename, 'w') as pyfile: - pyfile.write(filecont) - utsp = importlib.import_module('three_and_four') - methods = ['four_plus_four', 'three_plus_three'] - with open(pyfilename, 'r') as pyfile: - res, error_methods = srccoms_extract(pyfile, ['three_plus_three'], - methods) - self.assertTrue(res) - - expectedFile = os.path.join("samplecode_temp", - "four_plus_four_example.py") - self.assertTrue(os.path.exists(expectedFile)) - os.remove(expectedFile) - self.assertFalse(os.path.exists(expectedFile)) - - # https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/fluid/layers/ops.py # why? unabled to use the ast module. emmmmm -- GitLab