未验证 提交 4a09c1a1 编写于 作者: R Ren Wei (任卫) 提交者: GitHub

run the sample codes added by `add_sample_code` in ops.py (#31863)

* skip paddle.Tensor.<lambda>

* some file may not exists. such as version.py, it's generated by setup.py

* debug mode

* add unittests for sampcd_processor.py

* add test cases for sampcd_processor

* add test cases for sampcd_processor

* add testcases

* add test cases

* add testcases

* add testcases

* refactor, add testcases

* add import

* all files map to pool. dont split manually

* __all__ += another list

* add testcases

* add testcases

* handle个锤子啊

* this line should not removed

https://github.com/wadefelix/Paddle/commit/882e7f7c3be6c2415f58550f82be338b84f0c0ef#diff-cb0679475bf60202fd803ae05b9146989437c3f787d1502616be6c71c69d0fb1

* print -> logger

* regulate the logging infomation

* regulate the logging infomation

* logger to file

* logger

* threads or subprocesses number config

* follow the good code style

don't touch wlist.json

* run test_sampcd_processor.py, it's a unittest for sampcd_processor.py

* update unittest for sampcd_processor.py

test=document_fix
上级 0624ea56
...@@ -53,6 +53,7 @@ API_FILES=("CMakeLists.txt" ...@@ -53,6 +53,7 @@ API_FILES=("CMakeLists.txt"
"python/paddle/fluid/tests/unittests/white_list/check_op_sequence_batch_1_input_white_list.py" "python/paddle/fluid/tests/unittests/white_list/check_op_sequence_batch_1_input_white_list.py"
"python/paddle/fluid/tests/unittests/white_list/no_grad_set_white_list.py" "python/paddle/fluid/tests/unittests/white_list/no_grad_set_white_list.py"
"tools/wlist.json" "tools/wlist.json"
"tools/sampcd_processor.py"
"paddle/scripts/paddle_build.bat" "paddle/scripts/paddle_build.bat"
"tools/windows/run_unittests.sh" "tools/windows/run_unittests.sh"
"tools/parallel_UT_rule.py" "tools/parallel_UT_rule.py"
...@@ -79,6 +80,12 @@ function add_failed(){ ...@@ -79,6 +80,12 @@ function add_failed(){
echo_list="${echo_list[@]}$1" echo_list="${echo_list[@]}$1"
} }
function run_test_sampcd_processor() {
CUR_PWD=$(pwd)
cd ${PADDLE_ROOT}/tools
python test_sampcd_processor.py
cd ${CUR_PWD}
}
if [[ $git_files -gt 19 || $git_count -gt 999 ]];then if [[ $git_files -gt 19 || $git_count -gt 999 ]];then
echo_line="You must have Dianhai approval for change 20+ files or add than 1000+ lines of content.\n" echo_line="You must have Dianhai approval for change 20+ files or add than 1000+ lines of content.\n"
...@@ -136,6 +143,9 @@ for API_FILE in ${API_FILES[*]}; do ...@@ -136,6 +143,9 @@ for API_FILE in ${API_FILES[*]}; do
elif [ "${API_FILE}" == "tools/wlist.json" ];then elif [ "${API_FILE}" == "tools/wlist.json" ];then
echo_line="You must have one TPM (jzhang533) approval for the api whitelist for the tools/wlist.json.\n" echo_line="You must have one TPM (jzhang533) approval for the api whitelist for the tools/wlist.json.\n"
check_approval 1 29231 check_approval 1 29231
elif [ "${API_FILE}" == "tools/sampcd_processor.py" ];then
echo_line="test_sampcd_processor.py will be executed for changed sampcd_processor.py.\n"
run_test_sampcd_processor
elif [ "${API_FILE}" == "python/paddle/distributed/fleet/__init__.py" ]; then elif [ "${API_FILE}" == "python/paddle/distributed/fleet/__init__.py" ]; then
echo_line="You must have (fuyinno4 (Recommend), raindrops2sea) approval for ${API_FILE} changes" echo_line="You must have (fuyinno4 (Recommend), raindrops2sea) approval for ${API_FILE} changes"
check_approval 1 35824027 38231817 check_approval 1 35824027 38231817
......
此差异已折叠。
#! python
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import os
import tempfile
import shutil
import sys
import importlib
from sampcd_processor import find_all
from sampcd_processor import check_indent
from sampcd_processor import sampcd_extract_and_run
from sampcd_processor import single_defcom_extract
from sampcd_processor import srccoms_extract
from sampcd_processor import get_api_md5
from sampcd_processor import get_incrementapi
from sampcd_processor import get_wlist
class Test_find_all(unittest.TestCase):
def test_find_none(self):
self.assertEqual(0, len(find_all('hello', 'world')))
def test_find_one(self):
self.assertListEqual([0], find_all('hello', 'hello'))
def test_find_two(self):
self.assertListEqual([1, 15],
find_all(' hello, world; hello paddle!', 'hello'))
class Test_check_indent(unittest.TestCase):
def test_no_indent(self):
self.assertEqual(0, check_indent('hello paddle'))
def test_indent_4_spaces(self):
self.assertEqual(4, check_indent(' hello paddle'))
def test_indent_1_tab(self):
self.assertEqual(4, check_indent("\thello paddle"))
class Test_sampcd_extract_and_run(unittest.TestCase):
def setUp(self):
if not os.path.exists('samplecode_temp/'):
os.mkdir('samplecode_temp/')
def test_run_a_defs_samplecode(self):
comments = """
Examples:
.. code-block:: python
print(1+1)
"""
funcname = 'one_plus_one'
res, name, msg = sampcd_extract_and_run(comments, funcname)
self.assertTrue(res)
self.assertEqual(funcname, name)
def test_run_a_def_no_code(self):
comments = """
placeholder
"""
funcname = 'one_plus_one'
res, name, msg = sampcd_extract_and_run(comments, funcname)
self.assertFalse(res)
self.assertEqual(funcname, name)
def test_run_a_def_raise_expection(self):
comments = """
placeholder
Examples:
.. code-block:: python
print(1/0)
"""
funcname = 'one_plus_one'
res, name, msg = sampcd_extract_and_run(comments, funcname)
self.assertFalse(res)
self.assertEqual(funcname, name)
class Test_single_defcom_extract(unittest.TestCase):
def test_extract_from_func(self):
defstr = '''
import os
def foo():
"""
foo is a function.
"""
pass
def bar():
pass
'''
comm = single_defcom_extract(
2, defstr.splitlines(True), is_class_begin=False)
self.assertEqual(" foo is a function.\n", comm)
pass
def test_extract_from_func_with_no_docstring(self):
defstr = '''
import os
def bar():
pass
'''
comm = single_defcom_extract(
2, defstr.splitlines(True), is_class_begin=False)
self.assertEqual('', comm)
pass
def test_extract_from_class(self):
defstr = r'''
import os
class Foo():
"""
Foo is a class.
second line.
"""
pass
def bar():
pass
def foo():
pass
'''
comm = single_defcom_extract(
2, defstr.splitlines(True), is_class_begin=True)
rcomm = """ Foo is a class.
second line.
"""
self.assertEqual(rcomm, comm)
pass
def test_extract_from_class_with_no_docstring(self):
defstr = '''
import os
class Foo():
pass
def bar():
pass
def foo():
pass
'''
comm = single_defcom_extract(
0, defstr.splitlines(True), is_class_begin=True)
self.assertEqual('', comm)
class Test_get_api_md5(unittest.TestCase):
def setUp(self):
self.api_pr_spec_filename = os.path.abspath(
os.path.join(os.getcwd(), "..", 'paddle/fluid/API_PR.spec'))
with open(self.api_pr_spec_filename, 'w') as f:
f.write("\n".join([
"""one_plus_one (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'md5sum of one_plus_one'))""",
"""two_plus_two (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'md5sum of two_plus_two'))""",
"""three_plus_three (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'md5sum of three_plus_three'))""",
"""four_plus_four (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'md5sum of four_plus_four'))""",
]))
def tearDown(self):
os.remove(self.api_pr_spec_filename)
pass
def test_get_api_md5(self):
res = get_api_md5('paddle/fluid/API_PR.spec')
self.assertEqual("'md5sum of one_plus_one'", res['one_plus_one'])
self.assertEqual("'md5sum of two_plus_two'", res['two_plus_two'])
self.assertEqual("'md5sum of three_plus_three'",
res['three_plus_three'])
self.assertEqual("'md5sum of four_plus_four'", res['four_plus_four'])
class Test_get_incrementapi(unittest.TestCase):
def setUp(self):
self.api_pr_spec_filename = os.path.abspath(
os.path.join(os.getcwd(), "..", 'paddle/fluid/API_PR.spec'))
with open(self.api_pr_spec_filename, 'w') as f:
f.write("\n".join([
"""one_plus_one (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'md5sum of one_plus_one'))""",
"""two_plus_two (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'md5sum of two_plus_two'))""",
"""three_plus_three (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'md5sum of three_plus_three'))""",
"""four_plus_four (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'md5sum of four_plus_four'))""",
]))
self.api_dev_spec_filename = os.path.abspath(
os.path.join(os.getcwd(), "..", 'paddle/fluid/API_DEV.spec'))
with open(self.api_dev_spec_filename, 'w') as f:
f.write("\n".join([
"""one_plus_one (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'md5sum of one_plus_one'))""",
]))
self.api_diff_spec_filename = os.path.abspath(
os.path.join(os.getcwd(), "dev_pr_diff_api.spec"))
def tearDown(self):
os.remove(self.api_pr_spec_filename)
os.remove(self.api_dev_spec_filename)
os.remove(self.api_diff_spec_filename)
def test_it(self):
get_incrementapi()
with open(self.api_diff_spec_filename, 'r') as f:
lines = f.readlines()
self.assertCountEqual(
["two_plus_two\n", "three_plus_three\n", "four_plus_four\n"],
lines)
class Test_get_wlist(unittest.TestCase):
def setUp(self):
self.tmpDir = tempfile.mkdtemp()
self.wlist_filename = os.path.join(self.tmpDir, 'wlist.json')
with open(self.wlist_filename, 'w') as f:
f.write(r'''
{
"wlist_dir":[
{
"name":"../python/paddle/fluid/contrib",
"annotation":""
},
{
"name":"../python/paddle/verison.py",
"annotation":""
}
],
"wlist_api":[
{
"name":"xxxxx",
"annotation":"not a real api, just for example"
}
],
"wlist_temp_api":[
"to_tensor",
"save_persistables@dygraph/checkpoint.py"
],
"gpu_not_white":[
"deformable_conv"
]
}
''')
def tearDown(self):
os.remove(self.wlist_filename)
shutil.rmtree(self.tmpDir)
def test_get_wlist(self):
wlist, wlist_file, gpu_not_white = get_wlist(self.wlist_filename)
self.assertCountEqual(
["xxxxx", "to_tensor",
"save_persistables@dygraph/checkpoint.py"], wlist)
self.assertCountEqual([
"../python/paddle/fluid/contrib",
"../python/paddle/verison.py",
], wlist_file)
self.assertCountEqual(["deformable_conv"], gpu_not_white)
class Test_srccoms_extract(unittest.TestCase):
def setUp(self):
self.tmpDir = tempfile.mkdtemp()
sys.path.append(self.tmpDir)
self.api_pr_spec_filename = os.path.abspath(
os.path.join(os.getcwd(), "..", 'paddle/fluid/API_PR.spec'))
with open(self.api_pr_spec_filename, 'w') as f:
f.write("\n".join([
"""one_plus_one (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', "one_plus_one"))""",
"""two_plus_two (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', "two_plus_two"))""",
"""three_plus_three (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', "three_plus_three"))""",
"""four_plus_four (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', "four_plus_four"))""",
]))
def tearDown(self):
sys.path.remove(self.tmpDir)
shutil.rmtree(self.tmpDir)
os.remove(self.api_pr_spec_filename)
def test_from_ops_py(self):
filecont = '''
def add_sample_code(obj, docstr):
pass
__unary_func__ = [
'exp',
]
__all__ = []
__all__ += __unary_func__
__all__ += ['one_plus_one']
def exp():
pass
add_sample_code(globals()["exp"], r"""
Examples:
.. code-block:: python
import paddle
x = paddle.to_tensor([-0.4, -0.2, 0.1, 0.3])
out = paddle.exp(x)
print(out)
# [0.67032005 0.81873075 1.10517092 1.34985881]
""")
def one_plus_one():
return 1+1
one_plus_one.__doc__ = """
placeholder
Examples:
.. code-block:: python
print(1+1)
"""
__all__ += ['two_plus_two']
def two_plus_two():
return 2+2
add_sample_code(globals()["two_plus_two"], """
Examples:
.. code-block:: python
print(2+2)
""")
'''
pyfilename = os.path.join(self.tmpDir, 'ops.py')
with open(pyfilename, 'w') as pyfile:
pyfile.write(filecont)
self.assertTrue(os.path.exists(pyfilename))
utsp = importlib.import_module('ops')
print('testing srccoms_extract from ops.py')
methods = ['one_plus_one', 'two_plus_two', 'exp']
# os.remove("samplecode_temp/" "one_plus_one_example.py")
self.assertFalse(
os.path.exists("samplecode_temp/"
"one_plus_one_example.py"))
with open(pyfilename, 'r') as pyfile:
res, error_methods = srccoms_extract(pyfile, [], methods)
self.assertTrue(res)
self.assertTrue(
os.path.exists("samplecode_temp/"
"one_plus_one_example.py"))
os.remove("samplecode_temp/" "one_plus_one_example.py")
self.assertTrue(
os.path.exists("samplecode_temp/"
"two_plus_two_example.py"))
os.remove("samplecode_temp/" "two_plus_two_example.py")
self.assertTrue(os.path.exists("samplecode_temp/" "exp_example.py"))
os.remove("samplecode_temp/" "exp_example.py")
def test_from_not_ops_py(self):
filecont = '''
__all__ = [
'one_plus_one'
]
def one_plus_one():
"""
placeholder
Examples:
.. code-block:: python
print(1+1)
"""
return 1+1
'''
pyfilename = os.path.join(self.tmpDir, 'opo.py')
with open(pyfilename, 'w') as pyfile:
pyfile.write(filecont)
utsp = importlib.import_module('opo')
methods = ['one_plus_one']
with open(pyfilename, 'r') as pyfile:
res, error_methods = srccoms_extract(pyfile, [], methods)
self.assertTrue(res)
self.assertTrue(
os.path.exists("samplecode_temp/"
"one_plus_one_example.py"))
os.remove("samplecode_temp/" "one_plus_one_example.py")
def test_with_empty_wlist(self):
"""
see test_from_ops_py
"""
pass
def test_with_wlist(self):
filecont = '''
__all__ = [
'four_plus_four',
'three_plus_three'
]
def four_plus_four():
"""
placeholder
Examples:
.. code-block:: python
print(4+4)
"""
return 4+4
def three_plus_three():
"""
placeholder
Examples:
.. code-block:: python
print(3+3)
"""
return 3+3
'''
pyfilename = os.path.join(self.tmpDir, 'three_and_four.py')
with open(pyfilename, 'w') as pyfile:
pyfile.write(filecont)
utsp = importlib.import_module('three_and_four')
methods = ['four_plus_four', 'three_plus_three']
with open(pyfilename, 'r') as pyfile:
res, error_methods = srccoms_extract(pyfile, ['three_plus_three'],
methods)
self.assertTrue(res)
self.assertTrue(
os.path.exists("samplecode_temp/four_plus_four_example.py"))
os.remove("samplecode_temp/" "four_plus_four_example.py")
self.assertFalse(
os.path.exists("samplecode_temp/three_plus_three_example.py"))
# https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/fluid/layers/ops.py
# why? unabled to use the ast module. emmmmm
if __name__ == '__main__':
unittest.main()
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册