diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index 0b6b006bbb244188ac69c0218738fe3ef3bc9b49..42960c0cfb56fc27cb0b6924f2ed29150344bf89 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -562,6 +562,7 @@ function generate_upstream_develop_api_spec() { } function generate_api_spec() { + set -e spec_kind=$2 if [ "$spec_kind" != "PR" ] && [ "$spec_kind" != "DEV" ]; then echo "Not supported $2" @@ -572,7 +573,8 @@ function generate_api_spec() { cd ${PADDLE_ROOT}/build/.check_api_workspace virtualenv .${spec_kind}_env source .${spec_kind}_env/bin/activate - pip install ${PADDLE_ROOT}/build/python/dist/*whl + pip install -r ${PADDLE_ROOT}/python/requirements.txt + pip --no-cache-dir install ${PADDLE_ROOT}/build/python/dist/*whl spec_path=${PADDLE_ROOT}/paddle/fluid/API_${spec_kind}.spec python ${PADDLE_ROOT}/tools/print_signatures.py paddle > $spec_path diff --git a/python/requirements.txt b/python/requirements.txt index a78079650a982685b7c3d3fba97ed211d6d6d190..9473e604cc7d7df3fe4de847f9ef71cca1c76237 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,3 +1,4 @@ +opencv-python<=4.2.0.32 requests>=2.20.0 numpy>=1.12, <=1.16.4 ; python_version<"3.5" numpy>=1.12 ; python_version>="3.5" diff --git a/tools/count_api_without_core_ops.py b/tools/count_api_without_core_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..99e84074158ad7ddbdb91148b53cc3433f03f3f8 --- /dev/null +++ b/tools/count_api_without_core_ops.py @@ -0,0 +1,187 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import importlib +import inspect +import collections +import sys +import pydoc +import hashlib +import six +import functools + +__all__ = ['get_apis_with_and_without_core_ops', ] + +# APIs that should not be printed into API.spec +omitted_list = [ + "paddle.fluid.LoDTensor.set", # Do not know why it should be omitted + "paddle.fluid.io.ComposeNotAligned", + "paddle.fluid.io.ComposeNotAligned.__init__", +] + + +def md5(doc): + hash = hashlib.md5() + hash.update(str(doc).encode('utf-8')) + return hash.hexdigest() + + +def split_with_and_without_core_ops(member, cur_name): + if cur_name in omitted_list: + return + + if inspect.isclass(member): + pass + else: + try: + source = inspect.getsource(member) + if source.find('append_op') != -1: + if source.find('core.ops') != -1: + api_with_ops.append(cur_name) + else: + api_without_ops.append(cur_name) + except: + # If getsource failed (pybind API or function inherit from father class), just skip + pass + + +def get_md5_of_func(member, cur_name): + if cur_name in omitted_list: + return + + doc_md5 = md5(member.__doc__) + + if inspect.isclass(member): + pass + else: + try: + source = inspect.getsource(member) + func_dict[cur_name] = md5(source) + except: + # If getsource failed (pybind API or function inherit from father class), just skip + pass + + +def visit_member(parent_name, member, func): + cur_name = ".".join([parent_name, member.__name__]) + if inspect.isclass(member): + func(member, cur_name) + for name, value in inspect.getmembers(member): + if hasattr(value, '__name__') and (not name.startswith("_") or + name == "__init__"): + visit_member(cur_name, value, func) + elif inspect.ismethoddescriptor(member): + return + elif callable(member): + func(member, cur_name) + elif inspect.isgetsetdescriptor(member): + return + else: + raise RuntimeError("Unsupported generate signature of member, type {0}". + format(str(type(member)))) + + +def is_primitive(instance): + int_types = (int, long) if six.PY2 else (int, ) + pritimitive_types = int_types + (float, str) + if isinstance(instance, pritimitive_types): + return True + elif isinstance(instance, (list, tuple, set)): + for obj in instance: + if not is_primitive(obj): + return False + + return True + else: + return False + + +def visit_all_module(mod, visited, func): + mod_name = mod.__name__ + if mod_name != 'paddle' and not mod_name.startswith('paddle.'): + return + + if mod_name.startswith('paddle.fluid.core'): + return + + if mod in visited: + return + + visited.add(mod) + + for member_name in ( + name + for name in (mod.__all__ if hasattr(mod, "__all__") else dir(mod)) + if not name.startswith("_")): + instance = getattr(mod, member_name, None) + if instance is None: + continue + + if is_primitive(instance): + continue + + if not hasattr(instance, "__name__"): + continue + + if inspect.ismodule(instance): + visit_all_module(instance, visited, func) + else: + visit_member(mod.__name__, instance, func) + + +def get_apis_with_and_without_core_ops(modules): + global api_with_ops, api_without_ops + api_with_ops = [] + api_without_ops = [] + for m in modules: + visit_all_module( + importlib.import_module(m), set(), split_with_and_without_core_ops) + return api_with_ops, api_without_ops + + +def get_api_source_desc(modules): + global func_dict + func_dict = collections.OrderedDict() + for m in modules: + visit_all_module(importlib.import_module(m), set(), get_md5_of_func) + return func_dict + + +if __name__ == "__main__": + if len(sys.argv) > 1: + modules = sys.argv[2].split(",") + if sys.argv[1] == '-c': + api_with_ops, api_without_ops = get_apis_with_and_without_core_ops( + modules) + + print('api_with_ops:', len(api_with_ops)) + print('\n'.join(api_with_ops)) + print('\n==============\n') + print('api_without_ops:', len(api_without_ops)) + print('\n'.join(api_without_ops)) + + if sys.argv[1] == '-p': + func_dict = get_api_source_desc(modules) + for name in func_dict: + print(name, func_dict[name]) + + else: + print("""Usage: + 1. Count and list all operator-raleated APIs that contains append_op but not core.ops.xx. + python ./count_api_without_core_ops.py -c paddle + 2. Print api and the md5 of source code of the api. + python ./count_api_without_core_ops.py -p paddle + """)