diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index a0e630818d853ff5d09a08d01c6cac9e01ce3de6..a40c1487c707de814e2a3c65938606aa3e0024b9 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -831,11 +831,6 @@ function generate_api_spec() { awk -F '(' '{print $NF}' $spec_path >${spec_path}.doc awk -F '(' '{$NF="";print $0}' $spec_path >${spec_path}.api - if [ "$1" == "cp35-cp35m" ] || [ "$1" == "cp36-cp36m" ] || [ "$1" == "cp37-cp37m" ] || [ "$1" == "cp38-cp38" ] || [ "$1" == "cp39-cp39" ]; then - # Use sed to make python2 and python3 sepc keeps the same - sed -i 's/arg0: str/arg0: unicode/g' $spec_path - sed -i "s/\(.*Transpiler.*\).__init__ (ArgSpec(args=\['self'].*/\1.__init__ /g" $spec_path - fi python ${PADDLE_ROOT}/tools/diff_use_default_grad_op_maker.py \ ${PADDLE_ROOT}/paddle/fluid/op_use_default_grad_maker_${spec_kind}.spec @@ -2050,7 +2045,7 @@ function exec_samplecode_test() { if [ "$1" = "cpu" ] ; then python sampcd_processor.py cpu; example_error=$? elif [ "$1" = "gpu" ] ; then - python sampcd_processor.py --threads=16 --full-test gpu; example_error=$? + python sampcd_processor.py --threads=16 gpu; example_error=$? fi if [ "$example_error" != "0" ];then echo "Code instance execution failed" >&2 @@ -2164,9 +2159,15 @@ function main() { check_sequence_op_unittest generate_api_spec ${PYTHON_ABI:-""} "PR" set +e + example_info_gpu="" + example_code_gpu=0 + if [ "${WITH_GPU}" == "ON" ] ; then + example_info_gpu=$(exec_samplecode_test gpu) + example_code_gpu=$? + fi example_info=$(exec_samplecode_test cpu) example_code=$? - summary_check_problems $check_style_code $example_code "$check_style_info" "$example_info" + summary_check_problems $check_style_code $[${example_code_gpu} + ${example_code}] "$check_style_info" "${example_info_gpu}\n${example_info}" assert_api_spec_approvals ;; build) diff --git a/python/paddle/fluid/layers/ops.py b/python/paddle/fluid/layers/ops.py index a6ab50df08cc159d9b236eea84dc98863d7b7fbb..cc5c327b974f7477df2f861169ba9b6312d3bfcd 100755 --- a/python/paddle/fluid/layers/ops.py +++ b/python/paddle/fluid/layers/ops.py @@ -87,26 +87,26 @@ for _OP in set(__activations_noattr__): _new_OP = _OP if _OP in __deprecated_func_name__: _new_OP = __deprecated_func_name__[_OP] - func = generate_activation_fn(_OP) - func = deprecated( - since="2.0.0", update_to="paddle.nn.functional.%s" % (_new_OP))(func) - globals()[_OP] = func + _func = generate_activation_fn(_OP) + _func = deprecated( + since="2.0.0", update_to="paddle.nn.functional.%s" % (_new_OP))(_func) + globals()[_OP] = _func for _OP in set(__unary_func__): _new_OP = _OP if _OP in __deprecated_func_name__: _new_OP = __deprecated_func_name__[_OP] - func = generate_activation_fn(_OP) - func = deprecated(since="2.0.0", update_to="paddle.%s" % (_new_OP))(func) - globals()[_OP] = func + _func = generate_activation_fn(_OP) + _func = deprecated(since="2.0.0", update_to="paddle.%s" % (_new_OP))(_func) + globals()[_OP] = _func for _OP in set(__inplace_unary_func__): _new_OP = _OP if _OP in __deprecated_func_name__: _new_OP = __deprecated_func_name__[_OP] - func = generate_inplace_fn(_OP) - func = deprecated(since="2.0.0", update_to="paddle.%s" % (_new_OP))(func) - globals()[_OP] = func + _func = generate_inplace_fn(_OP) + _func = deprecated(since="2.0.0", update_to="paddle.%s" % (_new_OP))(_func) + globals()[_OP] = _func add_sample_code(globals()["sigmoid"], r""" Examples: diff --git a/tools/check_api_approvals.sh b/tools/check_api_approvals.sh index 97d97e8c0a26ad06eca8a8736e28992e69623222..19f07b5fa4be8d464682049c7f969bf745d8c5b5 100644 --- a/tools/check_api_approvals.sh +++ b/tools/check_api_approvals.sh @@ -38,7 +38,11 @@ function add_failed(){ api_spec_diff=`python ${PADDLE_ROOT}/tools/diff_api.py ${PADDLE_ROOT}/paddle/fluid/API_DEV.spec.api ${PADDLE_ROOT}/paddle/fluid/API_PR.spec.api` -if [ "$api_spec_diff" != "" ]; then +ops_func_in_diff=$(echo ${api_spec_diff} | grep '\bpaddle\.fluid\.layers\.ops\.func\b') +linenum=$(echo ${api_spec_diff} | wc -l | sed 's/[[:space:]]//g') +if [ "${linenum}" = "3" -a "${ops_func_in_diff}" != "" ] ; then + echo "skip paddle.fluid.layers.ops.func" +elif [ "$api_spec_diff" != "" ]; then echo_line="You must have one RD (XiaoguangHu01 or lanxianghit) and one TPM (saxon-zh or jzhang533 or dingjiaweiww or Heeenrrry or TCChenlong) approval for the api change for the management reason of API interface.\n" check_approval 1 46782768 47554610 echo_line="" @@ -46,14 +50,17 @@ if [ "$api_spec_diff" != "" ]; then fi api_doc_spec_diff=`python ${PADDLE_ROOT}/tools/diff_api.py ${PADDLE_ROOT}/paddle/fluid/API_DEV.spec.doc ${PADDLE_ROOT}/paddle/fluid/API_PR.spec.doc` -if [ "$api_doc_spec_diff" != "" ]; then +linenum=$(echo ${api_doc_spec_diff} | wc -l | sed 's/[[:space:]]//g') +if [ "${linenum}" = "3" -a "${ops_func_in_diff}" != "" ] ; then + echo "skip paddle.fluid.layers.ops.func for doc diff" +elif [ "$api_doc_spec_diff" != "" ]; then echo_line="You must have one TPM (saxon-zh or jzhang533 or dingjiaweiww or Heeenrrry or TCChenlong) approval for the api change for the management reason of API document.\n" check_approval 1 2870059 29231 23093488 28379894 11935832 fi -api_spec_diff=`python ${PADDLE_ROOT}/tools/check_api_source_without_core_ops.py ${PADDLE_ROOT}/paddle/fluid/API_DEV.source.md5 ${PADDLE_ROOT}/paddle/fluid/API_PR.source.md5` -if [ "$api_spec_diff" != "" ]; then - echo_line="APIs without core.ops: \n${api_spec_diff}\n" +api_src_spec_diff=`python ${PADDLE_ROOT}/tools/check_api_source_without_core_ops.py ${PADDLE_ROOT}/paddle/fluid/API_DEV.source.md5 ${PADDLE_ROOT}/paddle/fluid/API_PR.source.md5` +if [ "$api_src_spec_diff" != "" ]; then + echo_line="APIs without core.ops: \n${api_src_spec_diff}\n" echo_line="${echo_line}You must have one RD (zhiqiu (Recommend) or phlrain) approval for the api change for the opreator-related api without 'core.ops'.\n" echo_line="${echo_line}For more details, please click [https://github.com/PaddlePaddle/Paddle/wiki/paddle_api_development_manual.md]\n" check_approval 1 6888866 43953930 @@ -84,10 +91,16 @@ if [ -n "${echo_list}" ];then echo -e "${echo_list[@]}" echo "There are ${failed_num} approved errors." echo "****************" -fi -python ${PADDLE_ROOT}/tools/diff_api.py ${PADDLE_ROOT}/paddle/fluid/API_DEV.spec ${PADDLE_ROOT}/paddle/fluid/API_PR.spec -python ${PADDLE_ROOT}/tools/check_op_register_type.py ${PADDLE_ROOT}/paddle/fluid/OP_TYPE_DEV.spec ${PADDLE_ROOT}/paddle/fluid/OP_TYPE_PR.spec -if [ -n "${echo_list}" ]; then + # L40 L48 L62 has fetch the result out. + if [ "${api_spec_diff}" != "" ] ; then + echo "api_spec_diff: ${api_spec_diff}" + fi + if [ "${api_doc_spec_diff}" != "" ] ; then + echo "api_doc_spec_diff: ${api_doc_spec_diff}" + fi + if [ "${op_type_spec_diff}" != "" ] ; then + echo "op_type_spec_diff: ${op_type_spec_diff}" + fi exit 6 fi diff --git a/tools/print_signatures.py b/tools/print_signatures.py index 3f0a3e834f3d32455289567a15993a7b526d5397..b96ddcf549ec70f4b687cc2f77b703bd1eff34fc 100644 --- a/tools/print_signatures.py +++ b/tools/print_signatures.py @@ -29,6 +29,7 @@ import platform import functools import pkgutil import logging +import argparse import paddle member_dict = collections.OrderedDict() @@ -80,7 +81,9 @@ def is_primitive(instance): ErrorSet = set() IdSet = set() skiplist = [ - 'paddle.vision.datasets.DatasetFolderImageFolder', 'paddle.truncdigamma' + 'paddle.vision.datasets.DatasetFolderImageFolder', + 'paddle.truncdigamma', + 'paddle.fluid.layers.ops.func', ] @@ -100,9 +103,11 @@ def visit_all_module(mod): if hasattr(mod, "__all__"): member_names += mod.__all__ for member_name in member_names: - if member_name.startswith('__'): + if member_name.startswith('_'): continue cur_name = mod_name + '.' + member_name + if cur_name in skiplist: + continue try: instance = getattr(mod, member_name) if inspect.ismodule(instance): @@ -157,7 +162,8 @@ def get_all_api(root_path='paddle', attr="__all__"): logger.info('%s: collected %d apis, %d distinct apis.', attr, api_counter, len(api_info_dict)) - return [api_info['all_names'][0] for api_info in api_info_dict.values()] + return [(sorted(list(api_info['all_names']))[0], md5(api_info['docstring'])) + for api_info in api_info_dict.values()] def insert_api_into_dict(full_name, gen_doc_anno=None): @@ -185,6 +191,7 @@ def insert_api_into_dict(full_name, gen_doc_anno=None): "id": fc_id, "object": obj, "type": type(obj).__name__, + "docstring": '', } docstr = inspect.getdoc(obj) if docstr: @@ -229,15 +236,49 @@ def get_all_api_from_modulelist(): return member_dict +def parse_args(): + """ + Parse input arguments + """ + parser = argparse.ArgumentParser(description='Print Apis Signatures') + parser.add_argument('--debug', dest='debug', action="store_true") + parser.add_argument( + '--method', + dest='method', + type=str, + default='from_modulelist', + help="using get_all_api or from_modulelist") + parser.add_argument( + 'module', type=str, help='module', default='paddle') # not used + + if len(sys.argv) == 1: + args = parser.parse_args(['paddle']) + return args + # parser.print_help() + # sys.exit(1) + + args = parser.parse_args() + return args + + if __name__ == '__main__': - get_all_api_from_modulelist() + args = parse_args() + + if args.method == 'from_modulelist': + get_all_api_from_modulelist() + for name in member_dict: + print(name, member_dict[name]) + elif args.method == 'get_all_api': + api_signs = get_all_api() + for api_sign in api_signs: + print("{0} ({0}, ('document', '{1}'))".format(api_sign[0], api_sign[ + 1])) - for name in member_dict: - print(name, member_dict[name]) if len(ErrorSet) == 0: sys.exit(0) - for erroritem in ErrorSet: - print( - "Error, new function {} is unreachable".format(erroritem), - file=sys.stderr) - sys.exit(1) + else: + for erroritem in ErrorSet: + print( + "Error, new function {} is unreachable".format(erroritem), + file=sys.stderr) + sys.exit(1) diff --git a/tools/sampcd_processor.py b/tools/sampcd_processor.py index 07f112a5614a325103bed47efb379664ecc8e355..5acf9dc7d7670f47f129bb1004aa571098096d9b 100644 --- a/tools/sampcd_processor.py +++ b/tools/sampcd_processor.py @@ -443,7 +443,7 @@ def get_filenames(full_test=False): import paddle whl_error = [] if full_test: - get_full_api() + get_full_api_from_pr_spec() else: get_incrementapi() all_sample_code_filenames = {} @@ -513,7 +513,20 @@ def get_full_api_by_walk(): from print_signatures import get_all_api apilist = get_all_api() with open(API_DIFF_SPEC_FN, 'w') as f: - f.write("\n".join(apilist)) + f.write("\n".join([ai[0] for ai in apilist])) + + +def get_full_api_from_pr_spec(): + """ + get all the apis + """ + global API_PR_SPEC_FN, API_DIFF_SPEC_FN ## readonly + pr_api = get_api_md5(API_PR_SPEC_FN) + if len(pr_api): + with open(API_DIFF_SPEC_FN, 'w') as f: + f.write("\n".join(pr_api.keys())) + else: + get_full_api_by_walk() def get_incrementapi():