From 50f885fdaff0802ffbf0c109b9250e2870219ac7 Mon Sep 17 00:00:00 2001 From: zhiboniu <31800336+zhiboniu@users.noreply.github.com> Date: Mon, 21 Jun 2021 16:50:17 +0800 Subject: [PATCH] add new api ci check file (#33609) --- python/paddle/__init__.py | 2 +- python/paddle/vision/datasets/__init__.py | 2 +- tools/print_signatures.py | 170 +++++----------------- 3 files changed, 42 insertions(+), 132 deletions(-) diff --git a/python/paddle/__init__.py b/python/paddle/__init__.py index a3b01573b62..c81ee72d7f2 100755 --- a/python/paddle/__init__.py +++ b/python/paddle/__init__.py @@ -501,7 +501,7 @@ __all__ = [ # noqa 'log10', 'concat', 'check_shape', - 'trunc' + 'trunc', 'digamma', 'standard_normal' ] diff --git a/python/paddle/vision/datasets/__init__.py b/python/paddle/vision/datasets/__init__.py index 3ee7503e279..a9673aae21e 100644 --- a/python/paddle/vision/datasets/__init__.py +++ b/python/paddle/vision/datasets/__init__.py @@ -22,7 +22,7 @@ from .cifar import Cifar100 # noqa: F401 from .voc2012 import VOC2012 # noqa: F401 __all__ = [ #noqa - 'DatasetFolder' + 'DatasetFolder', 'ImageFolder', 'MNIST', 'FashionMNIST', diff --git a/tools/print_signatures.py b/tools/print_signatures.py index a63343782a0..3f0a3e834f3 100644 --- a/tools/print_signatures.py +++ b/tools/print_signatures.py @@ -62,96 +62,6 @@ def md5(doc): return md5sum -def get_functools_partial_spec(func): - func_str = func.func.__name__ - args = func.args - keywords = func.keywords - return '{}(args={}, keywords={})'.format(func_str, args, keywords) - - -def format_spec(spec): - args = spec.args - varargs = spec.varargs - keywords = spec.keywords - defaults = spec.defaults - if defaults is not None: - defaults = list(defaults) - for idx, item in enumerate(defaults): - if not isinstance(item, functools.partial): - continue - - defaults[idx] = get_functools_partial_spec(item) - - defaults = tuple(defaults) - - return 'ArgSpec(args={}, varargs={}, keywords={}, defaults={})'.format( - args, varargs, keywords, defaults) - - -def queue_dict(member, cur_name): - if cur_name != 'paddle': - try: - eval(cur_name) - except (AttributeError, NameError, SyntaxError) as e: - print( - "Error({}) occurred when `eval({})`, discard it.".format( - str(e), cur_name), - file=sys.stderr) - return - - if (inspect.isclass(member) or inspect.isfunction(member) or - inspect.ismethod(member)) and hasattr( - member, '__module__') and hasattr(member, '__name__'): - args = member.__module__ + "." + member.__name__ - try: - eval(args) - except (AttributeError, NameError, SyntaxError) as e: - print( - "Error({}) occurred when `eval({})`, discard it for {}.".format( - str(e), args, cur_name), - file=sys.stderr) - return - else: - try: - args = inspect.getargspec(member) - has_type_error = False - except TypeError: # special for PyBind method - args = " ".join([ - line.strip() for line in pydoc.render_doc(member).split('\n') - if "->" in line - ]) - has_type_error = True - - if not has_type_error: - args = format_spec(args) - - doc_md5 = md5(member.__doc__) - member_dict[cur_name] = "({}, ('document', '{}'))".format(args, doc_md5) - - -def visit_member(parent_name, member, member_name=None): - if member_name: - cur_name = ".".join([parent_name, member_name]) - else: - cur_name = ".".join([parent_name, member.__name__]) - if inspect.isclass(member): - queue_dict(member, cur_name) - for name, value in inspect.getmembers(member): - if hasattr(value, '__name__') and not name.startswith("_"): - visit_member(cur_name, value) - elif inspect.ismethoddescriptor(member): - return - elif inspect.isbuiltin(member): - return - elif callable(member): - queue_dict(member, cur_name) - elif inspect.isgetsetdescriptor(member): - return - else: - raise RuntimeError("Unsupported generate signature of member, type {0}". - format(str(type(member)))) - - def is_primitive(instance): int_types = (int, ) pritimitive_types = int_types + (float, str) @@ -167,6 +77,13 @@ def is_primitive(instance): return False +ErrorSet = set() +IdSet = set() +skiplist = [ + 'paddle.vision.datasets.DatasetFolderImageFolder', 'paddle.truncdigamma' +] + + def visit_all_module(mod): mod_name = mod.__name__ if mod_name != 'paddle' and not mod_name.startswith('paddle.'): @@ -177,37 +94,36 @@ def visit_all_module(mod): if mod in visited_modules: return - visited_modules.add(mod) + + member_names = dir(mod) if hasattr(mod, "__all__"): - member_names = (name for name in mod.__all__ - if not name.startswith("_")) - elif mod_name == 'paddle': - member_names = dir(mod) - else: - return + member_names += mod.__all__ for member_name in member_names: - instance = getattr(mod, member_name, None) - if instance is None: + if member_name.startswith('__'): continue - - if is_primitive(instance): - continue - - if not hasattr(instance, "__name__"): - continue - - if inspect.ismodule(instance): - visit_all_module(instance) - else: - if member_name != instance.__name__: - print( - "Found alias API, alias name is: {}, original name is: {}". - format(member_name, instance.__name__), - file=sys.stderr) - visit_member(mod.__name__, instance, member_name) + cur_name = mod_name + '.' + member_name + try: + instance = getattr(mod, member_name) + if inspect.ismodule(instance): + visit_all_module(instance) else: - visit_member(mod.__name__, instance) + doc_md5 = md5(instance.__doc__) + instance_id = id(instance) + if instance_id in IdSet: + continue + IdSet.add(instance_id) + member_dict[cur_name] = "({}, ('document', '{}'))".format( + cur_name, doc_md5) + if hasattr(instance, + '__name__') and member_name != instance.__name__: + print( + "Found alias API, alias name is: {}, original name is: {}". + format(member_name, instance.__name__), + file=sys.stderr) + except: + if not cur_name in ErrorSet and not cur_name in skiplist: + ErrorSet.add(cur_name) # all from gen_doc.py @@ -306,17 +222,7 @@ def process_module(m, attr="__all__"): def get_all_api_from_modulelist(): - modulelist = [ - paddle, paddle.amp, paddle.nn, paddle.nn.functional, - paddle.nn.initializer, paddle.nn.utils, paddle.static, paddle.static.nn, - paddle.io, paddle.jit, paddle.metric, paddle.distribution, - paddle.optimizer, paddle.optimizer.lr, paddle.regularizer, paddle.text, - paddle.utils, paddle.utils.download, paddle.utils.profiler, - paddle.utils.cpp_extension, paddle.sysconfig, paddle.vision, - paddle.distributed, paddle.distributed.fleet, - paddle.distributed.fleet.utils, paddle.distributed.parallel, - paddle.distributed.utils, paddle.callbacks, paddle.hub, paddle.autograd - ] + modulelist = [paddle] for m in modulelist: visit_all_module(m) @@ -324,10 +230,14 @@ def get_all_api_from_modulelist(): if __name__ == '__main__': - # modules = sys.argv[1].split(",") - # for m in modules: - # visit_all_module(importlib.import_module(m)) get_all_api_from_modulelist() for name in member_dict: print(name, member_dict[name]) + if len(ErrorSet) == 0: + sys.exit(0) + for erroritem in ErrorSet: + print( + "Error, new function {} is unreachable".format(erroritem), + file=sys.stderr) + sys.exit(1) -- GitLab