diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b58e1b0a4380dc27e84c43ab47aa51e4b276d0dd..c551cae5791f0ac48fb336cbe0eb1b4c9dbda734 100755 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,12 +33,6 @@ repos: hooks: - id: black files: (.*\.(py|pyi|bzl)|BUILD|.*\.BUILD|WORKSPACE)$ - # Temporary exclude, will be formatted in a separate PR - exclude: | - (?x)^( - python/paddle/fluid/tests/unittests/dygraph_to_static/test_error.py| - python/paddle/fluid/tests/unittests/dygraph_to_static/test_origin_info.py - )$ - repo: https://github.com/PyCQA/flake8 rev: 4.0.1 hooks: diff --git a/paddle/scripts/conda_build.py b/paddle/scripts/conda_build.py index f4411aa9fc794278422a697e5f3abe1d183e776d..12004460a50360f0b62ef9e182a092e1f16c76ce 100644 --- a/paddle/scripts/conda_build.py +++ b/paddle/scripts/conda_build.py @@ -22,17 +22,18 @@ import os def parse_args(): parser = argparse.ArgumentParser("conda build for paddlepaddle version") - parser.add_argument("--paddle_version", - type=str, - required=True, - help="paddle version for conda build.") + parser.add_argument( + "--paddle_version", + type=str, + required=True, + help="paddle version for conda build.", + ) args = parser.parse_args() return args class ConstantVar: - def __init__(self): self.build = r""" build: @@ -96,7 +97,10 @@ about: self.python39 = r" - python>=3.9, <3.10" self.python_version = [ - self.python36, self.python37, self.python38, self.python39 + self.python36, + self.python37, + self.python38, + self.python39, ] self.cuda101 = r""" @@ -112,9 +116,11 @@ about: - cudnn>=8.1, <8.2 """ - self.cuda_info = [(self.cuda101, "cuda10.1", ".post101"), - (self.cuda102, "cuda10.2", ""), - (self.cuda112, "cuda11.2", ".post112")] + self.cuda_info = [ + (self.cuda101, "cuda10.1", ".post101"), + (self.cuda102, "cuda10.2", ""), + (self.cuda112, "cuda11.2", ".post112"), + ] self.py_str = ["py36", "py37", "py38", "py39"] self.pip_end = ".whl --no-deps" self.pip_prefix_linux = "pip install /package/paddlepaddle" @@ -122,25 +128,36 @@ about: self.pip_gpu = "_gpu-" self.pip_cpu = "-" self.mac_pip = [ - "-cp36-cp36m-macosx_10_6_intel", "-cp37-cp37m-macosx_10_6_intel", - "-cp38-cp38-macosx_10_14_x86_64", "-cp39-cp39-macosx_10_14_x86_64" + "-cp36-cp36m-macosx_10_6_intel", + "-cp37-cp37m-macosx_10_6_intel", + "-cp38-cp38-macosx_10_14_x86_64", + "-cp39-cp39-macosx_10_14_x86_64", ] self.linux_pip = [ - "-cp36-cp36m-linux_x86_64", "-cp37-cp37m-linux_x86_64", - "-cp38-cp38-linux_x86_64", "-cp39-cp39-linux_x86_64" + "-cp36-cp36m-linux_x86_64", + "-cp37-cp37m-linux_x86_64", + "-cp38-cp38-linux_x86_64", + "-cp39-cp39-linux_x86_64", ] self.windows_pip = [ - "-cp36-cp36m-win_amd64", "-cp37-cp37m-win_amd64", - "-cp38-cp38-win_amd64", "-cp39-cp39-win_amd64" + "-cp36-cp36m-win_amd64", + "-cp37-cp37m-win_amd64", + "-cp38-cp38-win_amd64", + "-cp39-cp39-win_amd64", ] def meta_build_mac(var, python_str, paddle_version, build_var, build_name_str): - package_str = """ + package_str = ( + """ package: name: paddlepaddle - version: """ + paddle_version - requirement = var.requirement_build + python_str + var.requirement_run + python_str + version: """ + + paddle_version + ) + requirement = ( + var.requirement_build + python_str + var.requirement_run + python_str + ) meta_build = var.build + build_name_str meta_str = package_str + meta_build + requirement + var.test + var.about build_str = var.build_const + build_var @@ -153,23 +170,28 @@ package: f.write(build_str) -def meta_build_linux(var, - python_str, - paddle_version, - build_var, - build_name_str, - cuda_str=None): +def meta_build_linux( + var, python_str, paddle_version, build_var, build_name_str, cuda_str=None +): if cuda_str == None: - package_str = """ + package_str = ( + """ package: name: paddlepaddle - version: """ + paddle_version + version: """ + + paddle_version + ) else: - package_str = """ + package_str = ( + """ package: name: paddlepaddle-gpu - version: """ + paddle_version - requirement = var.requirement_build + python_str + var.requirement_run + python_str + version: """ + + paddle_version + ) + requirement = ( + var.requirement_build + python_str + var.requirement_run + python_str + ) meta_build = var.build + build_name_str meta_str = package_str + meta_build + requirement if not (cuda_str == None): @@ -186,24 +208,32 @@ package: f.write(build_str) -def meta_build_windows(var, - python_str, - paddle_version, - blt_var, - build_name_str, - cuda_str=None): +def meta_build_windows( + var, python_str, paddle_version, blt_var, build_name_str, cuda_str=None +): if cuda_str == None: - package_str = """ + package_str = ( + """ package: name: paddlepaddle - version: """ + paddle_version + version: """ + + paddle_version + ) else: - package_str = """ + package_str = ( + """ package: name: paddlepaddle-gpu - version: """ + paddle_version - - requirement = var.requirement_build + python_str + var.requirement_run_windows + python_str + version: """ + + paddle_version + ) + + requirement = ( + var.requirement_build + + python_str + + var.requirement_run_windows + + python_str + ) meta_build = var.build + build_name_str meta_str = package_str + meta_build + requirement @@ -223,12 +253,17 @@ package: def conda_build(paddle_version, var): sysstr = platform.system() - if (sysstr == "Windows"): + if sysstr == "Windows": os.system("mkdir paddle") os.chdir(r"./paddle") for i in range(len(var.python_version)): - blt_var = var.pip_prefix_windows + var.pip_cpu + paddle_version + var.windows_pip[ - i] + var.pip_end + blt_var = ( + var.pip_prefix_windows + + var.pip_cpu + + paddle_version + + var.windows_pip[i] + + var.pip_end + ) name = var.py_str[i] + "_cpu_windows" python_str = var.python_version[i] meta_build_windows(var, python_str, paddle_version, blt_var, name) @@ -237,21 +272,38 @@ def conda_build(paddle_version, var): for i in range(len(var.python_version)): for cuda_str in var.cuda_info: post = cuda_str[2] - blt_var = var.pip_prefix_windows + var.pip_gpu + paddle_version + post + var.windows_pip[ - i] + var.pip_end + blt_var = ( + var.pip_prefix_windows + + var.pip_gpu + + paddle_version + + post + + var.windows_pip[i] + + var.pip_end + ) name = var.py_str[i] + "_gpu_" + cuda_str[1] + "_windows" cuda_cudnn_str = cuda_str[0] python_str = var.python_version[i] - meta_build_windows(var, python_str, paddle_version, blt_var, - name, cuda_cudnn_str) + meta_build_windows( + var, + python_str, + paddle_version, + blt_var, + name, + cuda_cudnn_str, + ) os.system("conda build .") - elif (sysstr == "Linux"): + elif sysstr == "Linux": os.system("mkdir paddle") os.chdir(r"./paddle") for i in range(len(var.python_version)): - build_var = var.pip_prefix_linux + var.pip_cpu + paddle_version + var.linux_pip[ - i] + var.pip_end + build_var = ( + var.pip_prefix_linux + + var.pip_cpu + + paddle_version + + var.linux_pip[i] + + var.pip_end + ) name = var.py_str[i] + "_cpu_many_linux" python_str = var.python_version[i] meta_build_linux(var, python_str, paddle_version, build_var, name) @@ -260,23 +312,40 @@ def conda_build(paddle_version, var): for i in range(len(var.python_version)): for cuda_str in var.cuda_info: post = cuda_str[2] - build_var = var.pip_prefix_linux + var.pip_gpu + paddle_version + post + var.linux_pip[ - i] + var.pip_end + build_var = ( + var.pip_prefix_linux + + var.pip_gpu + + paddle_version + + post + + var.linux_pip[i] + + var.pip_end + ) name = var.py_str[i] + "_gpu_" + cuda_str[1] + "_many_linux" cuda_cudnn_str = cuda_str[0] python_str = var.python_version[i] - meta_build_linux(var, python_str, paddle_version, build_var, - name, cuda_cudnn_str) + meta_build_linux( + var, + python_str, + paddle_version, + build_var, + name, + cuda_cudnn_str, + ) os.system("conda build .") os.system("cd ..") - elif (sysstr == "Darwin"): + elif sysstr == "Darwin": os.system("mkdir paddle") os.chdir(r"./paddle") for i in range(len(var.python_version)): - build_var = var.pip_prefix_linux + var.pip_cpu + paddle_version + var.mac_pip[ - i] + var.pip_end + build_var = ( + var.pip_prefix_linux + + var.pip_cpu + + paddle_version + + var.mac_pip[i] + + var.pip_end + ) name = var.py_str[i] + "_mac" python_str = var.python_version[i] meta_build_mac(var, python_str, paddle_version, build_var, name) diff --git a/python/paddle/distributed/ps/utils/ps_program_builder.py b/python/paddle/distributed/ps/utils/ps_program_builder.py index f38ca4f016d1203cf490f9529cd8f47c03ec3eb9..cf24a962f253463b1b6428119abdc55496f8d49e 100755 --- a/python/paddle/distributed/ps/utils/ps_program_builder.py +++ b/python/paddle/distributed/ps/utils/ps_program_builder.py @@ -13,12 +13,13 @@ # limitations under the License. from .public import * # noqa: F403 -from paddle.distributed.fleet.base.private_helper_function import wait_server_ready +from paddle.distributed.fleet.base.private_helper_function import ( + wait_server_ready, +) from paddle.distributed.passes import new_pass class PsProgramBuilder(object): - def __init__(self, pass_ctx): self.pass_ctx = pass_ctx self.attrs = self.pass_ctx._attrs @@ -38,8 +39,9 @@ class PsProgramBuilder(object): self.launch_barrier = self.attrs['launch_barrier'] self.launch_barrier_flag = self.attrs['launch_barrier_flag'] - self.server_endpoints = self.attrs['role_maker']._get_pserver_endpoints( - ) + self.server_endpoints = self.attrs[ + 'role_maker' + ]._get_pserver_endpoints() def _build_trainer_desc(self): opt_info = self.loss.block.program._fleet_opt @@ -59,8 +61,9 @@ class PsProgramBuilder(object): ops = get_optimize_ops(self.attrs['origin_main_program']) if len(ops) == 0: return - add_lr_decay_table_pass = new_pass('add_lr_decay_table_pass', - self.attrs) + add_lr_decay_table_pass = new_pass( + 'add_lr_decay_table_pass', self.attrs + ) add_lr_decay_table_pass.apply([], [], self.pass_ctx) for op in ops: if op.type in ["sgd", "adam"]: @@ -73,8 +76,11 @@ class PsProgramBuilder(object): if self.attrs['is_worker']: self._build_trainer_programs() fluid.framework.switch_startup_program(self.cloned_startup) - print("fluid.default_startup_program: {}".format( - fluid.default_startup_program)) + print( + "fluid.default_startup_program: {}".format( + fluid.default_startup_program + ) + ) # print("ps_program_build before =", id(self.loss.block.program)) self._build_trainer_desc() self.loss.block.program = self.cloned_main @@ -89,16 +95,18 @@ class PsProgramBuilder(object): self._build_pserver_programs() self.loss.block.program = self.attrs['_main_server'] fluid.framework.switch_startup_program( - self.attrs['_startup_server']) + self.attrs['_startup_server'] + ) class GeoPsProgramBuilder(PsProgramBuilder): # 仅 CPU 模式 - def __init__(self, pass_ctx): super(GeoPsProgramBuilder, self).__init__(pass_ctx) if self.ps_mode != DistributedMode.GEO: - raise ValueError("ps mode: {} not matched {}", - format(self.ps_mode, "GeoPsProgramBuilder")) + raise ValueError( + "ps mode: {} not matched {}", + format(self.ps_mode, "GeoPsProgramBuilder"), + ) def _build_trainer_programs(self): append_send_ops_pass = new_pass("append_send_ops_pass", self.attrs) @@ -110,23 +118,25 @@ class GeoPsProgramBuilder(PsProgramBuilder): # 仅 CPU 模式 wait_server_ready(self.server_endpoints) def _build_pserver_programs(self): - add_listen_and_serv_pass = new_pass('add_listen_and_serv_pass', - self.attrs) - add_listen_and_serv_pass.apply([self.attrs['_main_server']], [None], - self.pass_ctx) + add_listen_and_serv_pass = new_pass( + 'add_listen_and_serv_pass', self.attrs + ) + add_listen_and_serv_pass.apply( + [self.attrs['_main_server']], [None], self.pass_ctx + ) return class NuPsProgramBuilder(PsProgramBuilder): - def __init__(self, pass_ctx): super(NuPsProgramBuilder, self).__init__(pass_ctx) if not self.attrs['local_sparse']: raise ValueError("No local sparse params") def _build_trainer_programs(self): - add_lr_decay_table_pass = new_pass("add_lr_decay_table_pass", - self.attrs) + add_lr_decay_table_pass = new_pass( + "add_lr_decay_table_pass", self.attrs + ) add_lr_decay_table_pass.apply([], [], self.pass_ctx) distributed_ops_pass = new_pass("distributed_ops_pass", self.attrs) @@ -135,20 +145,26 @@ class NuPsProgramBuilder(PsProgramBuilder): delete_optimizer_pass = new_pass("delete_optimizer_pass", self.attrs) delete_optimizer_pass.apply([self.cloned_main], [None], self.pass_ctx) - append_send_ops_pass = new_pass("append_send_ops_pass", - self.attrs) # fleet->PushDenseVarsAsync + append_send_ops_pass = new_pass( + "append_send_ops_pass", self.attrs + ) # fleet->PushDenseVarsAsync append_send_ops_pass.apply([self.cloned_main], [None], self.pass_ctx) - delete_extra_optimizer_pass = new_pass("delete_extra_optimizer_pass", - self.attrs) - delete_extra_optimizer_pass.apply([self.attrs['origin_main_program']], - [self.cloned_startup], self.pass_ctx) + delete_extra_optimizer_pass = new_pass( + "delete_extra_optimizer_pass", self.attrs + ) + delete_extra_optimizer_pass.apply( + [self.attrs['origin_main_program']], + [self.cloned_startup], + self.pass_ctx, + ) fake_init_ops_pass = new_pass("fake_init_ops_pass", self.attrs) fake_init_ops_pass.apply([None], [self.cloned_startup], self.pass_ctx) - append_send_ops_pass = new_pass("append_send_ops_pass", - self.attrs) # communicator->Send + append_send_ops_pass = new_pass( + "append_send_ops_pass", self.attrs + ) # communicator->Send append_send_ops_pass.apply([self.cloned_main], [None], self.pass_ctx) self.attrs['origin_main_program'] = self.cloned_main @@ -161,18 +177,23 @@ class NuPsProgramBuilder(PsProgramBuilder): class CpuSyncPsProgramBuilder(PsProgramBuilder): - def __init__(self, pass_ctx): super(CpuSyncPsProgramBuilder, self).__init__(pass_ctx) - if self.ps_mode != DistributedMode.SYNC and self.ps_mode != DistributedMode.ASYNC: - raise ValueError("ps mode: {} not matched {}", - format(self.ps_mode, "PsProgramBuilder")) + if ( + self.ps_mode != DistributedMode.SYNC + and self.ps_mode != DistributedMode.ASYNC + ): + raise ValueError( + "ps mode: {} not matched {}", + format(self.ps_mode, "PsProgramBuilder"), + ) def _build_trainer_programs(self): # print("build trainer program entry") # print("before ps program builder program:", self.cloned_main) - add_lr_decay_table_pass = new_pass("add_lr_decay_table_pass", - self.attrs) + add_lr_decay_table_pass = new_pass( + "add_lr_decay_table_pass", self.attrs + ) add_lr_decay_table_pass.apply([], [], self.pass_ctx) # print("before distributed op pass") @@ -185,10 +206,14 @@ class CpuSyncPsProgramBuilder(PsProgramBuilder): append_send_ops_pass = new_pass("append_send_ops_pass", self.attrs) append_send_ops_pass.apply([self.cloned_main], [None], self.pass_ctx) - delete_extra_optimizer_pass = new_pass("delete_extra_optimizer_pass", - self.attrs) - delete_extra_optimizer_pass.apply([self.attrs['origin_main_program']], - [self.cloned_startup], self.pass_ctx) + delete_extra_optimizer_pass = new_pass( + "delete_extra_optimizer_pass", self.attrs + ) + delete_extra_optimizer_pass.apply( + [self.attrs['origin_main_program']], + [self.cloned_startup], + self.pass_ctx, + ) fake_init_ops_pass = new_pass("fake_init_ops_pass", self.attrs) fake_init_ops_pass.apply([None], [self.cloned_startup], self.pass_ctx) @@ -204,7 +229,6 @@ class CpuSyncPsProgramBuilder(PsProgramBuilder): class CpuAsyncPsProgramBuilder(CpuSyncPsProgramBuilder): - def __init__(self, pass_ctx): super(CpuAsyncPsProgramBuilder, self).__init__(pass_ctx) @@ -212,15 +236,16 @@ class CpuAsyncPsProgramBuilder(CpuSyncPsProgramBuilder): opt_info = self.loss.block.program._fleet_opt opt_info = {} if opt_info is None else opt_info opt_info["trainer"] = opt_info.get("trainer", "DistMultiTrainer") - opt_info["device_worker"] = opt_info.get("device_worker", - "DownpourLite") + opt_info["device_worker"] = opt_info.get( + "device_worker", "DownpourLite" + ) pid = str(id(self.cloned_main)) program_configs = { pid: { 'pull_dense': [], 'push_dense': [], 'pull_sparse': [], - 'push_sparse': [] + 'push_sparse': [], } } dense_table_config = {} @@ -241,14 +266,14 @@ class CpuAsyncPsProgramBuilder(CpuSyncPsProgramBuilder): class GpuPsProgramBuilder(PsProgramBuilder): - def __init__(self, pass_ctx): super(GpuPsProgramBuilder, self).__init__(pass_ctx) def _build_trainer_programs(self): - add_lr_decay_table_pass = new_pass("add_lr_decay_table_pass", - self.attrs) + add_lr_decay_table_pass = new_pass( + "add_lr_decay_table_pass", self.attrs + ) add_lr_decay_table_pass.apply([], [], self.pass_ctx) distributed_ops_pass = new_pass("distributed_ops_pass", self.attrs) @@ -261,8 +286,9 @@ class GpuPsProgramBuilder(PsProgramBuilder): ps_gpu_pass.apply([self.cloned_main], [None], self.pass_ctx) ps_transpile_pass = new_pass("ps_transpile_pass", self.attrs) - ps_transpile_pass.apply([self.cloned_main], [self.cloned_startup], - self.pass_ctx) + ps_transpile_pass.apply( + [self.cloned_main], [self.cloned_startup], self.pass_ctx + ) self.attrs['origin_main_program'] = self.cloned_main self.attrs['origin_startup_program'] = self.cloned_startup @@ -274,13 +300,13 @@ class GpuPsProgramBuilder(PsProgramBuilder): class HeterAsyncPsProgramBuilder(PsProgramBuilder): - def __init__(self, pass_ctx): super(HeterAsyncPsProgramBuilder, self).__init__(pass_ctx) def _build_trainer_programs(self): - add_lr_decay_table_pass = new_pass("add_lr_decay_table_pass", - self.attrs) + add_lr_decay_table_pass = new_pass( + "add_lr_decay_table_pass", self.attrs + ) add_lr_decay_table_pass.apply([], [], self.pass_ctx) distributed_ops_pass = new_pass("distributed_ops_pass", self.attrs) @@ -292,29 +318,39 @@ class HeterAsyncPsProgramBuilder(PsProgramBuilder): append_send_ops_pass = new_pass("append_send_ops_pass", self.attrs) append_send_ops_pass.apply([self.cloned_main], [None], self.pass_ctx) - delete_extra_optimizer_pass = new_pass("delete_extra_optimizer_pass", - self.attrs) - delete_extra_optimizer_pass.apply([self.attrs['origin_main_program']], - [self.cloned_startup], self.pass_ctx) + delete_extra_optimizer_pass = new_pass( + "delete_extra_optimizer_pass", self.attrs + ) + delete_extra_optimizer_pass.apply( + [self.attrs['origin_main_program']], + [self.cloned_startup], + self.pass_ctx, + ) fake_init_ops_pass = new_pass("fake_init_ops_pass", self.attrs) fake_init_ops_pass.apply([None], [self.cloned_startup], self.pass_ctx) if self.is_heter_worker: split_heter_worker_ops_pass = new_pass( - "split_heter_worker_ops_pass", self.attrs) - split_heter_worker_ops_pass.apply([self.cloned_main], [None], - self.pass_ctx) + "split_heter_worker_ops_pass", self.attrs + ) + split_heter_worker_ops_pass.apply( + [self.cloned_main], [None], self.pass_ctx + ) else: - split_trainer_ops_pass = new_pass("split_trainer_ops_pass", - self.attrs) - split_trainer_ops_pass.apply([self.cloned_main], [None], - self.pass_ctx) - - set_heter_pipeline_opt_pass = new_pass('set_heter_pipeline_opt_pass', - self.attrs) - set_heter_pipeline_opt_pass.apply([self.cloned_main], - [self.cloned_startup], self.pass_ctx) + split_trainer_ops_pass = new_pass( + "split_trainer_ops_pass", self.attrs + ) + split_trainer_ops_pass.apply( + [self.cloned_main], [None], self.pass_ctx + ) + + set_heter_pipeline_opt_pass = new_pass( + 'set_heter_pipeline_opt_pass', self.attrs + ) + set_heter_pipeline_opt_pass.apply( + [self.cloned_main], [self.cloned_startup], self.pass_ctx + ) if self.launch_barrier and self.launch_barrier_flag: wait_server_ready(self.server_endpoints) @@ -325,58 +361,63 @@ class HeterAsyncPsProgramBuilder(PsProgramBuilder): if self.attrs['is_worker'] or self.attrs['is_heter_worker']: self._build_trainer_programs() ps_set_heter_pipeline_opt_pass = new_pass( - "set_heter_pipeline_opt_pass", self.attrs) - ps_set_heter_pipeline_opt_pass.apply([self.cloned_main], - [self.cloned_startup], - self.pass_ctx) + "set_heter_pipeline_opt_pass", self.attrs + ) + ps_set_heter_pipeline_opt_pass.apply( + [self.cloned_main], [self.cloned_startup], self.pass_ctx + ) elif self.attrs['is_server']: self._build_pserver_programs() self.loss.block.program = self.attrs['_main_server'] fluid.framework.switch_startup_program( - self.attrs['_startup_server']) + self.attrs['_startup_server'] + ) class FlPsProgramBuilder(HeterAsyncPsProgramBuilder): - def __init__(self, pass_ctx): super(FlPsProgramBuilder, self).__init__(pass_ctx) def _build_trainer_programs(self): _main_file = ps_log_root_dir + '0_fl_worker_main_program.prototxt' - #debug_program(_main_file, self.cloned_main) + # debug_program(_main_file, self.cloned_main) distributed_ops_pass = new_pass("distributed_ops_pass", self.attrs) distributed_ops_pass.apply([self.cloned_main], [None], self.pass_ctx) _main_file = ps_log_root_dir + '1_fl_worker_main_program.prototxt' - #debug_program(_main_file, self.cloned_main) + # debug_program(_main_file, self.cloned_main) delete_optimizer_pass = new_pass("delete_optimizer_pass", self.attrs) delete_optimizer_pass.apply([self.cloned_main], [None], self.pass_ctx) _main_file = ps_log_root_dir + '2_fl_worker_main_program.prototxt' - #debug_program(_main_file, self.cloned_main) + # debug_program(_main_file, self.cloned_main) append_send_ops_pass = new_pass("append_send_ops_pass", self.attrs) append_send_ops_pass.apply([self.cloned_main], [None], self.pass_ctx) _main_file = ps_log_root_dir + '3_fl_worker_main_program.prototxt' - #debug_program(_main_file, self.cloned_main) + # debug_program(_main_file, self.cloned_main) - delete_extra_optimizer_pass = new_pass("delete_extra_optimizer_pass", - self.attrs) - delete_extra_optimizer_pass.apply([self.attrs['origin_main_program']], - [self.cloned_startup], self.pass_ctx) + delete_extra_optimizer_pass = new_pass( + "delete_extra_optimizer_pass", self.attrs + ) + delete_extra_optimizer_pass.apply( + [self.attrs['origin_main_program']], + [self.cloned_startup], + self.pass_ctx, + ) _main_file = ps_log_root_dir + '4_fl_worker_main_program.prototxt' - #debug_program(_main_file, self.cloned_main) + # debug_program(_main_file, self.cloned_main) - #fake_init_ops_pass = new_pass("fake_init_ops_pass", self.attrs) - #fake_init_ops_pass.apply([None], [self.cloned_startup], self.pass_ctx) + # fake_init_ops_pass = new_pass("fake_init_ops_pass", self.attrs) + # fake_init_ops_pass.apply([None], [self.cloned_startup], self.pass_ctx) _main_file = ps_log_root_dir + '5_fl_worker_main_program.prototxt' - #debug_program(_main_file, self.cloned_main) + # debug_program(_main_file, self.cloned_main) split_trainer_ops_pass = new_pass("split_fl_ops_pass", self.attrs) split_trainer_ops_pass.apply([self.cloned_main], [None], self.pass_ctx) @@ -392,10 +433,12 @@ class FlPsProgramBuilder(HeterAsyncPsProgramBuilder): _main_file = ps_log_root_dir + '8_fl_B_main_program.prototxt' debug_program(_main_file, self.cloned_main) - set_heter_pipeline_opt_pass = new_pass('set_heter_pipeline_opt_pass', - self.attrs) - set_heter_pipeline_opt_pass.apply([self.cloned_main], - [self.cloned_startup], self.pass_ctx) + set_heter_pipeline_opt_pass = new_pass( + 'set_heter_pipeline_opt_pass', self.attrs + ) + set_heter_pipeline_opt_pass.apply( + [self.cloned_main], [self.cloned_startup], self.pass_ctx + ) self.attrs['origin_startup_program'] = self.cloned_startup self.attrs['origin_main_program'] = self.cloned_main @@ -403,13 +446,19 @@ class FlPsProgramBuilder(HeterAsyncPsProgramBuilder): if not self.is_heter_worker: _main_file = ps_log_root_dir + 'final_fl_A_main_program.prototxt' debug_program( - _main_file, self.attrs['origin_main_program']. - _heter_pipeline_opt['section_program']) + _main_file, + self.attrs['origin_main_program']._heter_pipeline_opt[ + 'section_program' + ], + ) else: _main_file = ps_log_root_dir + 'final_fl_B_main_program.prototxt' debug_program( - _main_file, self.attrs['origin_main_program']. - _heter_pipeline_opt['section_program']) + _main_file, + self.attrs['origin_main_program']._heter_pipeline_opt[ + 'section_program' + ], + ) return @@ -421,10 +470,14 @@ class FlPsProgramBuilder(HeterAsyncPsProgramBuilder): self._build_trainer_programs() fluid.framework.switch_startup_program(self.cloned_startup) fluid.framework.switch_main_program(self.cloned_main) - print("fluid.default_startup_program: {}".format( - fluid.default_startup_program()._heter_pipeline_opt)) + print( + "fluid.default_startup_program: {}".format( + fluid.default_startup_program()._heter_pipeline_opt + ) + ) else: self._build_pserver_programs() fluid.framework.switch_startup_program( - self.attrs['_startup_server']) + self.attrs['_startup_server'] + ) fluid.framework.switch_main_program(self.attrs['_main_server']) diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_build_cinn_pass_resnet.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_build_cinn_pass_resnet.py index 48679116ccf5b0cc87f9a29582a722fb72d38910..ebce394616027ab5efa44b380f443ca3d7c57e7e 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_build_cinn_pass_resnet.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_build_cinn_pass_resnet.py @@ -20,16 +20,17 @@ from model_zoo import resnet_model class TestBuildCINNPass(DistPassTestBase): - def init(self): self.atol = 0.5 self.rtol = 0.0 def apply_passes(self, main_prog, startup_prog): - pass_manager = PassManager([ - new_pass("build_cinn"), - new_pass("fuse_elewise_add_act"), - ]) + pass_manager = PassManager( + [ + new_pass("build_cinn"), + new_pass("fuse_elewise_add_act"), + ] + ) pass_manager.apply([main_prog], [startup_prog]) print(pass_manager.names) diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_build_cinn_pass_simple_net.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_build_cinn_pass_simple_net.py index 31bc9bd66d0325ba4713de6d17bb609c1e9914f9..293339ec81e628afe67590e93e54dcc2d2b8ce80 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_build_cinn_pass_simple_net.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_build_cinn_pass_simple_net.py @@ -20,16 +20,17 @@ from model_zoo import simple_net class TestBuildCINNPass(DistPassTestBase): - def init(self): self.atol = 0.0 self.rtol = 0.0 def apply_passes(self, main_prog, startup_prog): - pass_manager = PassManager([ - new_pass("build_cinn"), - new_pass("fuse_elewise_add_act"), - ]) + pass_manager = PassManager( + [ + new_pass("build_cinn"), + new_pass("fuse_elewise_add_act"), + ] + ) pass_manager.apply([main_prog], [startup_prog]) op_types = [op.type for op in main_prog.global_block().ops] self.assertTrue('cinn_launch' in op_types) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_build_strategy.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_build_strategy.py index 4746d355134e3de647f81c47877c436aa2686f43..439ca7720c16c9b42c99b6b5b1816f1dbddad234 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_build_strategy.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_build_strategy.py @@ -23,7 +23,6 @@ program_translator = ProgramTranslator() class TestResnetWithPass(unittest.TestCase): - def setUp(self): self.build_strategy = paddle.static.BuildStrategy() self.build_strategy.fuse_elewise_add_act_ops = True @@ -48,19 +47,24 @@ class TestResnetWithPass(unittest.TestCase): dy_pre, st_pre, rtol=1e-05, - err_msg='dy_pre:\n {}\n, st_pre: \n{}.'.format(dy_pre, st_pre)) + err_msg='dy_pre:\n {}\n, st_pre: \n{}.'.format(dy_pre, st_pre), + ) np.testing.assert_allclose( dy_jit_pre, st_pre, rtol=1e-05, err_msg='dy_jit_pre:\n {}\n, st_pre: \n{}.'.format( - dy_jit_pre, st_pre)) + dy_jit_pre, st_pre + ), + ) np.testing.assert_allclose( predictor_pre, st_pre, rtol=1e-05, err_msg='predictor_pre:\n {}\n, st_pre: \n{}.'.format( - predictor_pre, st_pre)) + predictor_pre, st_pre + ), + ) def test_resnet(self): static_loss = self.train(to_static=True) @@ -70,7 +74,9 @@ class TestResnetWithPass(unittest.TestCase): dygraph_loss, rtol=1e-05, err_msg='static_loss: {} \n dygraph_loss: {}'.format( - static_loss, dygraph_loss)) + static_loss, dygraph_loss + ), + ) self.verify_predict() def test_in_static_mode_mkldnn(self): @@ -83,9 +89,7 @@ class TestResnetWithPass(unittest.TestCase): class TestError(unittest.TestCase): - def test_type_error(self): - def foo(x): out = x + 1 return out diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_error.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_error.py index acd627c7fcf8fff03806a7774886a021cc509671..615e6bbf6fb1d4e7befeff1c3dcf8444a5347e25 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_error.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_error.py @@ -66,13 +66,13 @@ def func_decorated_by_other_2(): class LayerErrorInCompiletime(fluid.dygraph.Layer): - def __init__(self, fc_size=20): super(LayerErrorInCompiletime, self).__init__() self._linear = fluid.dygraph.Linear(fc_size, fc_size) @paddle.jit.to_static( - input_spec=[paddle.static.InputSpec(shape=[20, 20], dtype='float32')]) + input_spec=[paddle.static.InputSpec(shape=[20, 20], dtype='float32')] + ) def forward(self, x): y = self._linear(x) z = fluid.layers.fill_constant(shape=[1, 2], value=9, dtype="int") @@ -81,7 +81,6 @@ class LayerErrorInCompiletime(fluid.dygraph.Layer): class LayerErrorInCompiletime2(fluid.dygraph.Layer): - def __init__(self): super(LayerErrorInCompiletime2, self).__init__() @@ -93,7 +92,7 @@ class LayerErrorInCompiletime2(fluid.dygraph.Layer): """ NOTE: The next line has a tab. And this test to check the IndentationError when spaces and tabs are mixed. A tab here. - """ + """ # fmt: skip return @@ -108,7 +107,6 @@ def func_error_in_runtime_with_empty_line(x): class SuggestionErrorTestNet(paddle.nn.Layer): - def __init__(self): super(SuggestionErrorTestNet, self).__init__() self.inner_net = SuggestionErrorTestNet2() @@ -118,11 +116,10 @@ class SuggestionErrorTestNet(paddle.nn.Layer): return self.inner_net.forward(x) -class SuggestionErrorTestNet2(): - +class SuggestionErrorTestNet2: def __init__(self): super(SuggestionErrorTestNet2, self).__init__() - self.w = paddle.to_tensor([2.]) + self.w = paddle.to_tensor([2.0]) def forward(self, x): out = paddle.matmul(self.w, x) @@ -135,7 +132,6 @@ def func_suggestion_error_in_runtime(x): class TestFlags(unittest.TestCase): - def setUp(self): self.reset_flags_to_default() @@ -144,13 +140,15 @@ class TestFlags(unittest.TestCase): # 1. A flag to set whether to open the dygraph2static error reporting module os.environ[error.DISABLE_ERROR_ENV_NAME] = str( - error.DEFAULT_DISABLE_NEW_ERROR) + error.DEFAULT_DISABLE_NEW_ERROR + ) disable_error = int(os.getenv(error.DISABLE_ERROR_ENV_NAME, 999)) self.assertEqual(disable_error, 0) # 2. A flag to set whether to display the simplified error stack os.environ[error.SIMPLIFY_ERROR_ENV_NAME] = str( - error.DEFAULT_SIMPLIFY_NEW_ERROR) + error.DEFAULT_SIMPLIFY_NEW_ERROR + ) simplify_error = int(os.getenv(error.SIMPLIFY_ERROR_ENV_NAME, 999)) self.assertEqual(simplify_error, 1) @@ -167,7 +165,6 @@ class TestFlags(unittest.TestCase): class TestErrorBase(unittest.TestCase): - def setUp(self): self.set_input() self.set_func() @@ -188,20 +185,24 @@ class TestErrorBase(unittest.TestCase): def set_exception_type(self): raise NotImplementedError( - "Error test should implement set_exception_type") + "Error test should implement set_exception_type" + ) def set_message(self): raise NotImplementedError("Error test should implement set_message") def reset_flags_to_default(self): os.environ[error.DISABLE_ERROR_ENV_NAME] = str( - error.DEFAULT_DISABLE_NEW_ERROR) + error.DEFAULT_DISABLE_NEW_ERROR + ) os.environ[error.SIMPLIFY_ERROR_ENV_NAME] = str( - error.DEFAULT_SIMPLIFY_NEW_ERROR) + error.DEFAULT_SIMPLIFY_NEW_ERROR + ) def disable_new_error(self): os.environ[error.DISABLE_ERROR_ENV_NAME] = str( - 1 - error.DEFAULT_DISABLE_NEW_ERROR) + 1 - error.DEFAULT_DISABLE_NEW_ERROR + ) def _test_new_error_message(self, new_exception, disable_new_error=0): error_message = str(new_exception) @@ -242,7 +243,6 @@ class TestErrorBase(unittest.TestCase): # Situation 1: Call StaticLayer.__call__ to use Dynamic-to-Static class TestErrorStaticLayerCallInCompiletime(TestErrorBase): - def set_func(self): self.func = func_error_in_compile_time @@ -255,7 +255,8 @@ class TestErrorStaticLayerCallInCompiletime(TestErrorBase): def set_message(self): self.expected_message = [ 'File "{}", line 33, in func_error_in_compile_time'.format( - self.filepath), + self.filepath + ), 'inner_func()', 'File "{}", line 26, in inner_func'.format(self.filepath), 'def inner_func():', @@ -274,8 +275,8 @@ class TestErrorStaticLayerCallInCompiletime(TestErrorBase): class TestErrorStaticLayerCallInCompiletime_2( - TestErrorStaticLayerCallInCompiletime): - + TestErrorStaticLayerCallInCompiletime +): def set_func(self): self.func = func_error_in_compile_time_2 @@ -285,7 +286,8 @@ class TestErrorStaticLayerCallInCompiletime_2( def set_message(self): self.expected_message = [ 'File "{}", line 44, in func_error_in_compile_time_2'.format( - self.filepath), + self.filepath + ), 'def func_error_in_compile_time_2(x):', 'x = fluid.dygraph.to_variable(x)', 'x = fluid.layers.reshape(x, shape=[1, 2])', @@ -295,8 +297,8 @@ class TestErrorStaticLayerCallInCompiletime_2( class TestErrorStaticLayerCallInCompiletime_3( - TestErrorStaticLayerCallInCompiletime): - + TestErrorStaticLayerCallInCompiletime +): def setUp(self): self.reset_flags_to_default() self.set_func_call() @@ -309,7 +311,7 @@ class TestErrorStaticLayerCallInCompiletime_3( def set_message(self): self.expected_message = [ - 'File "{}", line 90, in forward'.format(self.filepath), + 'File "{}", line 89, in forward'.format(self.filepath), '@paddle.jit.to_static', 'def forward(self):', 'self.test_func()', @@ -325,7 +327,6 @@ class TestErrorStaticLayerCallInCompiletime_3( class TestErrorStaticLayerCallInRuntime(TestErrorStaticLayerCallInCompiletime): - def set_func(self): self.func = func_error_in_runtime @@ -335,7 +336,8 @@ class TestErrorStaticLayerCallInRuntime(TestErrorStaticLayerCallInCompiletime): def set_message(self): self.expected_message = [ 'File "{}", line 52, in func_error_in_runtime'.format( - self.filepath), + self.filepath + ), 'x = fluid.dygraph.to_variable(x)', 'two = fluid.layers.fill_constant(shape=[1], value=2, dtype="int32")', 'x = fluid.layers.reshape(x, shape=[1, two])', @@ -345,14 +347,14 @@ class TestErrorStaticLayerCallInRuntime(TestErrorStaticLayerCallInCompiletime): class TestErrorStaticLayerCallInRuntime2(TestErrorStaticLayerCallInRuntime): - def set_func(self): self.func = func_error_in_runtime_with_empty_line def set_message(self): self.expected_message = [ - 'File "{}", line 105, in func_error_in_runtime_with_empty_line'. - format(self.filepath), + 'File "{}", line 104, in func_error_in_runtime_with_empty_line'.format( + self.filepath + ), 'two = fluid.layers.fill_constant(shape=[1], value=2, dtype="int32")', 'x = fluid.layers.reshape(x, shape=[1, two])', '<--- HERE', @@ -362,29 +364,29 @@ class TestErrorStaticLayerCallInRuntime2(TestErrorStaticLayerCallInRuntime): # Situation 2: Call ProgramTranslator().get_output(...) to use Dynamic-to-Static class TestErrorGetOutputInCompiletime(TestErrorStaticLayerCallInCompiletime): - def set_func_call(self): self.func_call = lambda: self.prog_trans.get_output( - unwrap(self.func), self.input) - + unwrap(self.func), self.input + ) -class TestErrorGetOutputInCompiletime_2(TestErrorStaticLayerCallInCompiletime_2 - ): +class TestErrorGetOutputInCompiletime_2( + TestErrorStaticLayerCallInCompiletime_2 +): def set_func_call(self): self.func_call = lambda: self.prog_trans.get_output( - unwrap(self.func), self.input) + unwrap(self.func), self.input + ) class TestErrorGetOutputInRuntime(TestErrorStaticLayerCallInRuntime): - def set_func_call(self): self.func_call = lambda: self.prog_trans.get_output( - unwrap(self.func), self.input) + unwrap(self.func), self.input + ) class TestJitSaveInCompiletime(TestErrorBase): - def setUp(self): self.reset_flags_to_default() self.set_func_call() @@ -409,7 +411,8 @@ class TestJitSaveInCompiletime(TestErrorBase): def set_func_call(self): layer = LayerErrorInCompiletime() self.func_call = lambda: paddle.jit.save( - layer, path="./test_dy2stat_error/model") + layer, path="./test_dy2stat_error/model" + ) def test_error(self): self._test_raise_new_exception() @@ -419,21 +422,20 @@ class TestJitSaveInCompiletime(TestErrorBase): class TestSuggestionErrorInRuntime(TestErrorBase): - def set_func(self): self.func = func_suggestion_error_in_runtime def set_input(self): - self.input = paddle.to_tensor([2.]) + self.input = paddle.to_tensor([2.0]) def set_exception_type(self): self.exception_type = ValueError def set_message(self): self.expected_message = [ - 'File "{}", line 118, in forward'.format(self.filepath), + 'File "{}", line 116, in forward'.format(self.filepath), 'return self.inner_net.forward(x)', - 'File "{}", line 128, in forward'.format(self.filepath), + 'File "{}", line 125, in forward'.format(self.filepath), 'def forward(self, x):', 'out = paddle.matmul(self.w, x)', '<--- HERE', @@ -460,7 +462,6 @@ def func_ker_error(x): class TestKeyError(unittest.TestCase): - def test_key_error(self): paddle.disable_static() with self.assertRaises(error.Dy2StKeyError): @@ -476,7 +477,6 @@ def NpApiErr(): class TestNumpyApiErr(unittest.TestCase): - def test_numpy_api_err(self): with self.assertRaises(TypeError) as e: NpApiErr() @@ -490,11 +490,11 @@ class TestNumpyApiErr(unittest.TestCase): self.assertIn( "values will be changed to variables by dy2static, numpy api can not handle variables", - error_message) + error_message, + ) class test_set_state_dict_err_layer(paddle.nn.Layer): - def __init__(self): super(test_set_state_dict_err_layer, self).__init__() self.linear = paddle.nn.Linear(5, 2) @@ -514,11 +514,10 @@ class test_set_state_dict_err_layer(paddle.nn.Layer): class TestSetStateDictErr(unittest.TestCase): - def test_set_state_dict_err(self): with self.assertRaises(ValueError) as e: layer = test_set_state_dict_err_layer() - x = paddle.to_tensor([1., 2., 3., 4., 5.]) + x = paddle.to_tensor([1.0, 2.0, 3.0, 4.0, 5.0]) y = layer(x) new_exception = e.exception @@ -530,7 +529,8 @@ class TestSetStateDictErr(unittest.TestCase): self.assertIn( "This error might happens in dy2static, while calling 'set_state_dict' dynamicly in 'forward', which is not supported. If you only need call 'set_state_dict' once, move it to '__init__'.", - error_message) + error_message, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_origin_info.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_origin_info.py index 744cc40cdfb583b0517750dc1b34b681a68cf9e9..663c08ea28e3164ba8f7739177214809490aed67 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_origin_info.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_origin_info.py @@ -15,8 +15,19 @@ import sys import unittest -from paddle.fluid.dygraph.dygraph_to_static.ast_transformer import DygraphToStaticAst -from paddle.fluid.dygraph.dygraph_to_static.origin_info import Location, ORIGI_INFO, OriginInfo, attach_origin_info, create_and_update_origin_info_map, gast, inspect, unwrap +from paddle.fluid.dygraph.dygraph_to_static.ast_transformer import ( + DygraphToStaticAst, +) +from paddle.fluid.dygraph.dygraph_to_static.origin_info import ( + Location, + ORIGI_INFO, + OriginInfo, + attach_origin_info, + create_and_update_origin_info_map, + gast, + inspect, + unwrap, +) from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_func from paddle.fluid.dygraph.jit import declarative @@ -27,7 +38,6 @@ def simple_func(x): def nested_func(x): - def f1(a): return a @@ -47,7 +57,6 @@ def decorated_func2(x): class TestOriginInfo(unittest.TestCase): - def setUp(self): self.set_test_func() self.dygraph_func = unwrap(self.func) @@ -55,8 +64,9 @@ class TestOriginInfo(unittest.TestCase): self.source_code = inspect.getsource(self.dygraph_func) lines, self.start_lineno = inspect.getsourcelines(self.dygraph_func) lines = [line.strip("\n") for line in lines] - self.lines = [line for line in lines - if line != ""] # Delete empty lines + self.lines = [ + line for line in lines if line != "" + ] # Delete empty lines self.set_static_lineno() self.set_dygraph_info() @@ -77,8 +87,9 @@ class TestOriginInfo(unittest.TestCase): def set_origin_info_list(self, dygraph_ast): assert isinstance(dygraph_ast, gast.Module) self.transformed_node_list = [ - dygraph_ast.body[0], dygraph_ast.body[0].body[0], - dygraph_ast.body[0].body[1] + dygraph_ast.body[0], + dygraph_ast.body[0].body[0], + dygraph_ast.body[0].body[1], ] def _get_OriginInfo_map(self): @@ -91,8 +102,9 @@ class TestOriginInfo(unittest.TestCase): # step3 self.static_func, _ = ast_to_func(transformed_ast, self.dygraph_func) - info_map = create_and_update_origin_info_map(dygraph_ast, - self.static_func) + info_map = create_and_update_origin_info_map( + dygraph_ast, self.static_func + ) return info_map def test_origin_info_map(self): @@ -115,9 +127,12 @@ class TestOriginInfo(unittest.TestCase): code = self.lines[line_idx] origin_info = OriginInfo( Location(self.dygraph_filepath, dy_lineno, dy_col_offset), - self.dy_func_name[i], code) - self.assertEqual(str(origin_info_map[staic_loc.line_location]), - str(origin_info)) + self.dy_func_name[i], + code, + ) + self.assertEqual( + str(origin_info_map[staic_loc.line_location]), str(origin_info) + ) def test_attach_origin_info(self): dygraph_ast = gast.parse(self.source_code) @@ -144,7 +159,6 @@ class TestOriginInfo(unittest.TestCase): class TestOriginInfoWithNestedFunc(TestOriginInfo): - def set_test_func(self): self.func = nested_func @@ -154,23 +168,26 @@ class TestOriginInfoWithNestedFunc(TestOriginInfo): def set_dygraph_info(self): self.line_num = 5 self.line_index_list = [0, 1, 2, 3, 4] - self.dy_rel_lineno_list = [0, 2, 3, 5, 6] + self.dy_rel_lineno_list = [0, 1, 2, 4, 5] self.dy_abs_col_offset = [0, 4, 8, 4, 4] - self.dy_func_name = [self.dygraph_func.__name__] + \ - ["f1"] * 2 + \ - [self.dygraph_func.__name__] * 2 + self.dy_func_name = ( + [self.dygraph_func.__name__] + + ["f1"] * 2 + + [self.dygraph_func.__name__] * 2 + ) def set_origin_info_list(self, dygraph_ast): assert isinstance(dygraph_ast, gast.Module) self.transformed_node_list = [ - dygraph_ast.body[0], dygraph_ast.body[0].body[0], - dygraph_ast.body[0].body[0].body[0], dygraph_ast.body[0].body[1], - dygraph_ast.body[0].body[2] + dygraph_ast.body[0], + dygraph_ast.body[0].body[0], + dygraph_ast.body[0].body[0].body[0], + dygraph_ast.body[0].body[1], + dygraph_ast.body[0].body[2], ] class TestOriginInfoWithDecoratedFunc(TestOriginInfo): - def set_test_func(self): self.func = decorated_func @@ -205,7 +222,6 @@ class TestOriginInfoWithDecoratedFunc(TestOriginInfo): class TestOriginInfoWithDecoratedFunc2(TestOriginInfo): - def set_test_func(self): self.func = decorated_func2 diff --git a/python/paddle/fluid/tests/unittests/test_build_strategy_fusion_group_pass.py b/python/paddle/fluid/tests/unittests/test_build_strategy_fusion_group_pass.py index fa90b252baa93b8eee59d7e5360a7aaba32648c4..962a5d3327c78eeecb75f33cfba3f96dead9caf7 100644 --- a/python/paddle/fluid/tests/unittests/test_build_strategy_fusion_group_pass.py +++ b/python/paddle/fluid/tests/unittests/test_build_strategy_fusion_group_pass.py @@ -19,7 +19,6 @@ from test_eager_deletion_padding_rnn import RNNConfig, PaddingRNNTestBase class FusionGroupPaddingRNNTest(PaddingRNNTestBase): - def set_customed_config(self): self.build_strategy.enable_auto_fusion = True diff --git a/python/paddle/fluid/tests/unittests/test_pass_builder.py b/python/paddle/fluid/tests/unittests/test_pass_builder.py index 55b3d7985baaaf40f02622703f2802872cefc395..d8b29249a0847ccba82b163592e003ba3785738c 100644 --- a/python/paddle/fluid/tests/unittests/test_pass_builder.py +++ b/python/paddle/fluid/tests/unittests/test_pass_builder.py @@ -25,7 +25,6 @@ import tempfile class TestPassBuilder(unittest.TestCase): - def check_network_convergence(self, use_cuda, build_strategy=None): os.environ['CPU_NUM'] = str(4) main = fluid.Program() @@ -47,20 +46,22 @@ class TestPassBuilder(unittest.TestCase): feed_dict = {'image': image, 'label': label} train_cp = compiler.CompiledProgram(main).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy) + loss_name=loss.name, build_strategy=build_strategy + ) test_cp = compiler.CompiledProgram(test_program).with_data_parallel( loss_name=loss.name, build_strategy=build_strategy, - share_vars_from=train_cp) + share_vars_from=train_cp, + ) for i in range(5): _ = exe.run(train_cp, fetch_list=[loss.name], feed=feed_dict) - test_loss, = exe.run(test_cp, - fetch_list=[loss.name], - feed=feed_dict) - train_loss, = exe.run(train_cp, - fetch_list=[loss.name], - feed=feed_dict) + (test_loss,) = exe.run( + test_cp, fetch_list=[loss.name], feed=feed_dict + ) + (train_loss,) = exe.run( + train_cp, fetch_list=[loss.name], feed=feed_dict + ) avg_test_loss_val = np.array(test_loss).mean() if math.isnan(float(avg_test_loss_val)): @@ -70,32 +71,38 @@ class TestPassBuilder(unittest.TestCase): if math.isnan(float(avg_train_loss_val)): sys.exit("got NaN loss, training failed.") - np.testing.assert_allclose(train_loss, - test_loss, - rtol=1e-05, - atol=1e-08, - err_msg='Train loss: ' + - str(train_loss) + '\n Test loss:' + - str(test_loss)) + np.testing.assert_allclose( + train_loss, + test_loss, + rtol=1e-05, + atol=1e-08, + err_msg='Train loss: ' + + str(train_loss) + + '\n Test loss:' + + str(test_loss), + ) def test_parallel_testing_with_new_strategy(self): build_strategy = fluid.BuildStrategy() self.assertFalse(build_strategy.fuse_elewise_add_act_ops) build_strategy.fuse_elewise_add_act_ops = True - #FIXME: currently fuse_elewise_add_act_ops not compatible with below options + # FIXME: currently fuse_elewise_add_act_ops not compatible with below options build_strategy.enable_inplace = False build_strategy.memory_optimize = False pass_builder = build_strategy._finalize_strategy_and_create_passes() - self.assertTrue("fuse_elewise_add_act_pass" in - [p.type() for p in pass_builder.all_passes()]) + self.assertTrue( + "fuse_elewise_add_act_pass" + in [p.type() for p in pass_builder.all_passes()] + ) origin_len = len(pass_builder.all_passes()) viz_pass = pass_builder.append_pass("graph_viz_pass") self.assertEqual(origin_len + 1, len(pass_builder.all_passes())) - pass_builder.insert_pass(len(pass_builder.all_passes()), - "graph_viz_pass") + pass_builder.insert_pass( + len(pass_builder.all_passes()), "graph_viz_pass" + ) self.assertEqual(origin_len + 2, len(pass_builder.all_passes())) pass_builder.remove_pass(len(pass_builder.all_passes()) - 1) @@ -106,7 +113,8 @@ class TestPassBuilder(unittest.TestCase): self.check_network_convergence( use_cuda=core.is_compiled_with_cuda(), - build_strategy=build_strategy) + build_strategy=build_strategy, + ) try: os.stat(graph_viz_path) except os.error: diff --git a/python/paddle/fluid/tests/unittests/test_signal.py b/python/paddle/fluid/tests/unittests/test_signal.py index 5e363dd39513bc682503396a48dfee49bebf621e..6b485d56d79acb86d11b0bcc2bc1f9a867b9b4e0 100644 --- a/python/paddle/fluid/tests/unittests/test_signal.py +++ b/python/paddle/fluid/tests/unittests/test_signal.py @@ -865,18 +865,18 @@ class TestOverlapAddException(unittest.TestCase): @parameterize( (TEST_CASE_NAME, 'x', 'n_fft', 'hop_length', 'win_length', 'window', 'center', 'pad_mode', 'normalized', 'onesided'), [ - ('test_1d_input', rand_x(1, np.float64, shape=[160000]), - 512, None, None, get_window('hann', 512), True, 'reflect', False, True), - ('test_2d_input', rand_x(2, np.float64, shape=[1, 160000]), - 512, None, None, get_window('hann', 512), True, 'reflect', False, True), - ('test_hop_length', rand_x(2, np.float64, shape=[1, 160000]), - 512, 255, None, get_window('hann', 512), True, 'reflect', False, True), - ('test_win_length', rand_x(2, np.float64, shape=[1, 160000]), - 512, 255, 499, get_window('hann', 499), True, 'reflect', False, True), - ('test_window', rand_x(2, np.float64, shape=[1, 160000]), - 512, None, None, None, True, 'reflect', False, True), - ('test_center', rand_x(2, np.float64, shape=[1, 160000]), - 512, None, None, None, False, 'reflect', False, True), + ('test_1d_input', rand_x(1, np.float64, shape=[160000]), 512, + None, None, get_window('hann', 512), True, 'reflect', False, True), + ('test_2d_input', rand_x(2, np.float64, shape=[1, 160000]), 512, + None, None, get_window('hann', 512), True, 'reflect', False, True), + ('test_hop_length', rand_x(2, np.float64, shape=[1, 160000]), 512, + 255, None, get_window('hann', 512), True, 'reflect', False, True), + ('test_win_length', rand_x(2, np.float64, shape=[1, 160000]), 512, + 255, 499, get_window('hann', 499), True, 'reflect', False, True), + ('test_window', rand_x(2, np.float64, shape=[1, 160000]), 512, + None, None, None, True, 'reflect', False, True), + ('test_center', rand_x(2, np.float64, shape=[1, 160000]), 512, + None, None, None, False, 'reflect', False, True), ]) # fmt: skip class TestStft(unittest.TestCase): def test_stft(self): @@ -917,22 +917,22 @@ class TestStft(unittest.TestCase): @parameterize( (TEST_CASE_NAME, 'x', 'n_fft', 'hop_length', 'win_length', 'window', 'center', 'pad_mode', 'normalized', 'onesided', 'expect_exception'), [ - ('test_dims', rand_x(1, np.float64, shape=[1, 2, 3]), - 512, None, None, None, True, 'reflect', False, True, AssertionError), - ('test_hop_length', rand_x(1, np.float64, shape=[16000]), - 512, 0, None, None, True, 'reflect', False, True, AssertionError), - ('test_nfft1', rand_x(1, np.float64, shape=[16000]), - 0, None, None, None, True, 'reflect', False, True, AssertionError), - ('test_nfft2', rand_x(1, np.float64, shape=[16000]), - 16001, None, None, None, True, 'reflect', False, True, AssertionError), - ('test_win_length', rand_x(1, np.float64, shape=[16000]), - 512, None, 0, None, True, 'reflect', False, True, AssertionError), - ('test_win_length', rand_x(1, np.float64, shape=[16000]), - 512, None, 513, None, True, 'reflect', False, True, AssertionError), - ('test_pad_mode', rand_x(1, np.float64, shape=[16000]), - 512, None, None, None, True, 'nonsense', False, True, AssertionError), - ('test_complex_onesided', rand_x(1, np.float64, shape=[16000], complex=True), - 512, None, None, None, False, 'reflect', False, True, AssertionError), + ('test_dims', rand_x(1, np.float64, shape=[1, 2, 3]), 512, + None, None, None, True, 'reflect', False, True, AssertionError), + ('test_hop_length', rand_x(1, np.float64, shape=[16000]), 512, + 0, None, None, True, 'reflect', False, True, AssertionError), + ('test_nfft1', rand_x(1, np.float64, shape=[16000]), 0, + None, None, None, True, 'reflect', False, True, AssertionError), + ('test_nfft2', rand_x(1, np.float64, shape=[16000]), 16001, + None, None, None, True, 'reflect', False, True, AssertionError), + ('test_win_length', rand_x(1, np.float64, shape=[16000]), 512, + None, 0, None, True, 'reflect', False, True, AssertionError), + ('test_win_length', rand_x(1, np.float64, shape=[16000]), 512, + None, 513, None, True, 'reflect', False, True, AssertionError), + ('test_pad_mode', rand_x(1, np.float64, shape=[16000]), 512, + None, None, None, True, 'nonsense', False, True, AssertionError), + ('test_complex_onesided', rand_x(1, np.float64, shape=[16000], complex=True), 512, + None, None, None, False, 'reflect', False, True, AssertionError), ]) # fmt: skip class TestStftException(unittest.TestCase): def test_stft(self): @@ -959,20 +959,20 @@ class TestStftException(unittest.TestCase): @parameterize( (TEST_CASE_NAME, 'x', 'n_fft', 'hop_length', 'win_length', 'window', 'center', 'normalized', 'onesided', 'length', 'return_complex'), [ - ('test_2d_input', rand_x(2, np.float64, shape=[257, 471], complex=True), - 512, None, None, get_window('hann', 512), True, False, True, None, False), - ('test_3d_input', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), - 512, None, None, get_window('hann', 512), True, False, True, None, False), - ('test_hop_length', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), - 512, 99, None, get_window('hann', 512), True, False, True, None, False), - ('test_win_length', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), - 512, 99, 299, get_window('hann', 299), True, False, True, None, False), - ('test_window', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), - 512, None, None, None, True, False, True, None, False), - ('test_center', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), - 512, None, None, None, False, False, True, None, False), - ('test_length', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), - 512, None, None, None, False, False, True, 1888, False), + ('test_2d_input', rand_x(2, np.float64, shape=[257, 471], complex=True), 512, + None, None, get_window('hann', 512), True, False, True, None, False), + ('test_3d_input', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), 512, + None, None, get_window('hann', 512), True, False, True, None, False), + ('test_hop_length', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), 512, + 99, None, get_window('hann', 512), True, False, True, None, False), + ('test_win_length', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), 512, + 99, 299, get_window('hann', 299), True, False, True, None, False), + ('test_window', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), 512, + None, None, None, True, False, True, None, False), + ('test_center', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), 512, + None, None, None, False, False, True, None, False), + ('test_length', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), 512, + None, None, None, False, False, True, 1888, False), ]) # fmt: skip class TestIstft(unittest.TestCase): def test_istft(self): @@ -1013,30 +1013,30 @@ class TestIstft(unittest.TestCase): @parameterize( (TEST_CASE_NAME, 'x', 'n_fft', 'hop_length', 'win_length', 'window', 'center', 'normalized', 'onesided', 'length', 'return_complex', 'expect_exception'), [ - ('test_dims', rand_x(4, np.float64, shape=[1, 2, 3, 4], complex=True), - 512, None, None, get_window('hann', 512), True, False, True, None, False, AssertionError), - ('test_n_fft', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), - 257, None, None, get_window('hann', 512), True, False, True, None, False, AssertionError), - ('test_hop_length1', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), - 512, 0, None, get_window('hann', 512), True, False, True, None, False, AssertionError), - ('test_hop_length2', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), - 512, 513, None, get_window('hann', 512), True, False, True, None, False, AssertionError), - ('test_win_length1', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), - 512, None, 0, get_window('hann', 512), True, False, True, None, False, AssertionError), - ('test_win_length2', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), - 512, None, 513, get_window('hann', 512), True, False, True, None, False, AssertionError), - ('test_onesided1', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), - 20, None, None, get_window('hann', 512), True, False, True, None, False, AssertionError), - ('test_onesided2', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), - 256, None, None, None, True, False, False, None, False, AssertionError), - ('test_window', rand_x(3, np.float64, shape=[1, 512, 471], complex=True), - 512, None, 511, get_window('hann', 512), True, False, False, None, False, AssertionError), - ('test_return_complex1', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), - 512, None, None, get_window('hann', 512), True, False, True, None, True, AssertionError), - ('test_return_complex2', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), - 512, None, None, rand_x(1, np.float64, shape=[512], complex=True), True, False, True, None, False, AssertionError), - ('test_NOLA', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), - 512, 512, None, get_window('hann', 512), True, False, True, None, False, ValueError), + ('test_dims', rand_x(4, np.float64, shape=[1, 2, 3, 4], complex=True), 512, + None, None, get_window('hann', 512), True, False, True, None, False, AssertionError), + ('test_n_fft', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), 257, + None, None, get_window('hann', 512), True, False, True, None, False, AssertionError), + ('test_hop_length1', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), 512, + 0, None, get_window('hann', 512), True, False, True, None, False, AssertionError), + ('test_hop_length2', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), 512, + 513, None, get_window('hann', 512), True, False, True, None, False, AssertionError), + ('test_win_length1', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), 512, + None, 0, get_window('hann', 512), True, False, True, None, False, AssertionError), + ('test_win_length2', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), 512, + None, 513, get_window('hann', 512), True, False, True, None, False, AssertionError), + ('test_onesided1', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), 20, + None, None, get_window('hann', 512), True, False, True, None, False, AssertionError), + ('test_onesided2', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), 256, + None, None, None, True, False, False, None, False, AssertionError), + ('test_window', rand_x(3, np.float64, shape=[1, 512, 471], complex=True), 512, + None, 511, get_window('hann', 512), True, False, False, None, False, AssertionError), + ('test_return_complex1', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), 512, + None, None, get_window('hann', 512), True, False, True, None, True, AssertionError), + ('test_return_complex2', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), 512, + None, None, rand_x(1, np.float64, shape=[512], complex=True), True, False, True, None, False, AssertionError), + ('test_NOLA', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), 512, + 512, None, get_window('hann', 512), True, False, True, None, False, ValueError), ]) # fmt: skip class TestIstftException(unittest.TestCase): def test_istft(self): diff --git a/tools/analysis_build_time.py b/tools/analysis_build_time.py index e2cc454f5733cdc4365e5fdc9109624e914615ff..1ed9e7aa65d6b6324c80ed77a4d47dbe7a50a50c 100644 --- a/tools/analysis_build_time.py +++ b/tools/analysis_build_time.py @@ -26,10 +26,12 @@ def strToSecond(strTime): def getUsefulBuildTimeFile(filename): os.system( "grep -Po -- '-o .*' %s | grep ' elapsed' | grep -P -v '0:00.* elapse' > %s/tools/analysis_build_time" - % (filename, root_path)) + % (filename, root_path) + ) os.system( "grep -v -- '-o .*' %s |grep ' elapse' | grep -P -v '0:00.* elapse' >> %s/tools/analysis_build_time" - % (filename, root_path)) + % (filename, root_path) + ) def analysisBuildTime(): @@ -45,19 +47,24 @@ def analysisBuildTime(): buildFile = line.split(', ')[0].split(' ')[1] buildTime = line.split(', ')[1].split('elapsed')[0].strip() secondTime = strToSecond(buildTime) - os.system("echo %s, %s >> %s/tools/tempbuildTime.txt" % - (buildFile, secondTime, root_path)) + os.system( + "echo %s, %s >> %s/tools/tempbuildTime.txt" + % (buildFile, secondTime, root_path) + ) else: buildTime = line.split(', ')[1].split('elapsed')[0].strip() secondTime = strToSecond(buildTime) if secondTime > 30: - os.system("echo %s, %s >> %s/tools/tempbuildTime.txt" % - (line, secondTime, root_path)) + os.system( + "echo %s, %s >> %s/tools/tempbuildTime.txt" + % (line, secondTime, root_path) + ) except ValueError: print(line) os.system( - 'sort -n -k 2 -r %s/tools/tempbuildTime.txt > %s/tools/buildTime.txt' % - (root_path, root_path)) + 'sort -n -k 2 -r %s/tools/tempbuildTime.txt > %s/tools/buildTime.txt' + % (root_path, root_path) + ) analysisBuildTime() diff --git a/tools/dockerfile/build_scripts/manylinux1-check.py b/tools/dockerfile/build_scripts/manylinux1-check.py index 0d1a6df4eec98c72e493517d54fae7c416727d38..ac5c3008d2e574aa070beb70cffbf0bdd7a79434 100644 --- a/tools/dockerfile/build_scripts/manylinux1-check.py +++ b/tools/dockerfile/build_scripts/manylinux1-check.py @@ -18,12 +18,14 @@ def is_manylinux1_compatible(): # Only Linux, and only x86-64 / i686 from distutils.util import get_platform + if get_platform() not in ["linux-x86_64", "linux-i686"]: return False # Check for presence of _manylinux module try: import _manylinux + return bool(_manylinux.manylinux1_compatible) except (ImportError, AttributeError): # Fall through to heuristic check below @@ -62,9 +64,10 @@ def have_compatible_glibc(major, minimum_minor): import sys + if is_manylinux1_compatible(): - print("%s is manylinux1 compatible" % (sys.executable, )) + print("%s is manylinux1 compatible" % (sys.executable,)) sys.exit(0) else: - print("%s is NOT manylinux1 compatible" % (sys.executable, )) + print("%s is NOT manylinux1 compatible" % (sys.executable,)) sys.exit(1) diff --git a/tools/dockerfile/build_scripts/ssl-check.py b/tools/dockerfile/build_scripts/ssl-check.py index afef2812f3fb4e9298ec8ab2d97e790ecc455d1c..b44a5753c1a6f94ee71376779b083d47336170eb 100644 --- a/tools/dockerfile/build_scripts/ssl-check.py +++ b/tools/dockerfile/build_scripts/ssl-check.py @@ -21,22 +21,24 @@ import sys print("Testing SSL certificate checking for Python:", sys.version) -if (sys.version_info[:2] < (2, 7) or sys.version_info[:2] < (3, 4)): +if sys.version_info[:2] < (2, 7) or sys.version_info[:2] < (3, 4): print("This version never checks SSL certs; skipping tests") sys.exit(0) if sys.version_info[0] >= 3: from urllib.request import urlopen + EXC = OSError else: from urllib import urlopen + EXC = IOError -print("Connecting to %s should work" % (GOOD_SSL, )) +print("Connecting to %s should work" % (GOOD_SSL,)) urlopen(GOOD_SSL) print("...it did, yay.") -print("Connecting to %s should fail" % (BAD_SSL, )) +print("Connecting to %s should fail" % (BAD_SSL,)) try: urlopen(BAD_SSL) # If we get here then we failed: